Support large JSON files (#16211)

Use i64 internally for line/column numbers in JSON documents. To avoid breaking changes, existing accessors convert to i32, raising on overflow. New i64 accessors are added, marked experimental, and used internally.

Author: RX14

src/json.cr

@@ -120,16 +120,41 @@ module JSON
 
   # Exception thrown on a JSON parse error.
   class ParseException < Error
-    getter line_number : Int32
-    getter column_number : Int32
+    @line_number : Int64
+    @column_number : Int64
 
-    def initialize(message, @line_number, @column_number, cause = nil)
+    def line_number : Int32
+      @line_number.to_i32
+    end
+
+    def column_number : Int32
+      @column_number.to_i32
+    end
+
+    @[Experimental]
+    def line_number_i64 : Int64
+      @line_number
+    end
+
+    @[Experimental]
+    def column_number_i64
+      @column_number
+    end
+
+    def initialize(message, line_number, column_number, cause = nil)
+      @line_number = line_number.to_i64
+      @column_number = column_number.to_i64
       super "#{message} at line #{@line_number}, column #{@column_number}", cause
     end
 
     def location : {Int32, Int32}
       {line_number, column_number}
     end
+
+    @[Experimental]
+    def location_i64 : {Int64, Int64}
+      {line_number_i64, column_number_i64}
+    end
   end
 
   # Parses a JSON document as a `JSON::Any`.

src/json/from_json.cr

@@ -138,7 +138,7 @@ end
                        } %}
   def {{type.id}}.new(pull : JSON::PullParser)
     # TODO: use `PullParser#read?` instead
-    location = pull.location
+    location = pull.location_i64
     value =
       {% if type == "UInt64" || type == "UInt128" || type == "Int128" %}
         pull.read_raw
@@ -282,7 +282,7 @@ def NamedTuple.new(pull : JSON::PullParser)
       {% end %}
     {% end %}
 
-    location = pull.location
+    location = pull.location_i64
 
     pull.read_object do |key|
       case key
@@ -389,7 +389,7 @@ module Enum::ValueConverter(T)
 end
 
 def Union.new(pull : JSON::PullParser)
-  location = pull.location
+  location = pull.location_i64
 
   {% begin %}
     case pull.kind

src/json/lexer.cr

@@ -12,6 +12,9 @@ abstract class JSON::Lexer
   getter token : Token
   property skip : Bool
 
+  @line_number : Int64
+  @column_number : Int64
+
   def initialize
     @token = Token.new
     @line_number = 1

src/json/pull_parser.cr

@@ -104,7 +104,7 @@ class JSON::PullParser
     @raw_value = ""
     @object_stack = [] of ObjectStackKind
     @skip_count = 0
-    @location = {0, 0}
+    @location = {0_i64, 0_i64}
 
     next_token
     case token.kind
@@ -577,19 +577,39 @@ class JSON::PullParser
   end
 
   # Returns the current line number.
-  def line_number
+  @[Experimental]
+  def line_number_i64 : Int64
     @location[0]
   end
 
+  # Returns the current line number.
+  def line_number : Int32
+    @location[0].to_i32
+  end
+
   # Returns the current column number.
-  def column_number
+  @[Experimental]
+  def column_number_i64
     @location[1]
   end
 
+  # Returns the current column number.
+  def column_number
+    @location[1].to_i32
+  end
+
   # Returns the current location.
   #
   # The location is a tuple `{line number, column number}`.
   def location : Tuple(Int32, Int32)
+    {line_number, column_number}
+  end
+
+  # Returns the current location.
+  #
+  # The location is a tuple `{line number, column number}`.
+  @[Experimental]
+  def location_i64 : Tuple(Int64, Int64)
     @location
   end
 
@@ -654,12 +674,12 @@ class JSON::PullParser
   end
 
   private def next_token
-    @location = {@lexer.token.line_number, @lexer.token.column_number}
+    @location = {@lexer.token.line_number_i64, @lexer.token.column_number_i64}
     @lexer.next_token
   end
 
   private def next_token_expect_object_key
-    @location = {@lexer.token.line_number, @lexer.token.column_number}
+    @location = {@lexer.token.line_number_i64, @lexer.token.column_number_i64}
     @lexer.next_token_expect_object_key
   end
 

src/json/serialization.cr

@@ -210,14 +210,14 @@ module JSON
               %found{name} = false
             {% end %}
 
-            %location = pull.location
+            %location = pull.location_i64
             begin
               pull.read_begin_object
             rescue exc : ::JSON::ParseException
               raise ::JSON::SerializableError.new(exc.message, self.class.to_s, nil, *%location, exc)
             end
             until pull.kind.end_object?
-              %key_location = pull.location
+              %key_location = pull.location_i64
               key = pull.read_object_key
               case key
               {% for name, value in properties %}
@@ -421,7 +421,7 @@ module JSON
       {% end %}
 
       def self.new(pull : ::JSON::PullParser)
-        location = pull.location
+        location = pull.location_i64
 
         discriminator_value = nil
 
@@ -486,7 +486,7 @@ module JSON
     getter klass : String
     getter attribute : String?
 
-    def initialize(message : String?, @klass : String, @attribute : String?, line_number : Int32, column_number : Int32, cause)
+    def initialize(message : String?, @klass : String, @attribute : String?, line_number, column_number, cause)
       message = String.build do |io|
         io << message
         io << "\n  parsing "
@@ -497,7 +497,7 @@ module JSON
       end
       super(message, line_number, column_number, cause)
       if cause
-        @line_number, @column_number = cause.location
+        @line_number, @column_number = cause.location_i64
       end
     end
   end

src/json/token.cr

@@ -30,8 +30,35 @@ class JSON::Token
     raise ParseException.new(exc.message, line_number, column_number)
   end
 
-  property line_number : Int32
-  property column_number : Int32
+  @line_number : Int64
+  @column_number : Int64
+
+  def line_number : Int32
+    @line_number.to_i32
+  end
+
+  @[Experimental]
+  def line_number_i64 : Int64
+    @line_number
+  end
+
+  def line_number=(line_number)
+    @line_number = line_number.to_i64
+  end
+
+  def column_number : Int32
+    @column_number.to_i32
+  end
+
+  @[Experimental]
+  def column_number_i64
+    @column_number
+  end
+
+  def column_number=(column_number)
+    @column_number = column_number.to_i64
+  end
+
   property raw_value : String
 
   def initialize