WIP: preserve duplicate-key semantics in parse

.rubocop.yml

@@ -49,5 +49,8 @@ Metrics/AbcSize:
 Metrics/MethodLength:
   Max: 15
 
+Metrics/ParameterLists:
+  Max: 6
+
 Bundler/DuplicatedGem:
   Enabled: false

README.md

@@ -78,7 +78,7 @@ end
 
 # Result contains byte offsets, you need to be careful when working with non-binary strings
 emoji_json = '{"grin": "😁", "heart": "😍", "rofl": "🤣"}'
-begin_pos, end_pos, = JsonScanner.scan(emoji_json, [["heart"]]).first.first
+begin_pos, end_pos, = JsonScanner.scan(emoji_json, [["heart"]]).first.last
 emoji_json.byteslice(begin_pos...end_pos)
 # => "\"😍\""
 # Note: You most likely don't need the `quirks_mode` option unless you are using
@@ -92,7 +92,7 @@ JSON.parse(emoji_json.byteslice(begin_pos...end_pos), quirks_mode: true)
 # surrogate pairs), while dropping the quotes leaves those escape bytes
 # untouched. Pass the complete quoted slice to JSON.parse instead.
 escaped_json = '{"value":"\\"\\\\\\/\\b\\f\\n\\r\\t\\u0061\\uD83D\\uDE01"}'
-begin_pos, end_pos, = JsonScanner.scan(escaped_json, [["value"]]).first.first
+begin_pos, end_pos, = JsonScanner.scan(escaped_json, [["value"]]).first.last
 quoted_slice = escaped_json.byteslice(begin_pos...end_pos)
 quoted_slice.byteslice(1, quoted_slice.bytesize - 2)
 # => "\\\"\\\\\\/\\b\\f\\n\\r\\t\\u0061\\uD83D\\uDE01"
@@ -139,6 +139,30 @@ JsonScanner.parse('[1, 2, null, {"a": 42, "b": 33}, 5]', [[(1..2)], [3, "a"]])
 
 `JsonScanner.parse` supports almost the same options `JsonScanner.scan` does, except `with_path` and `with_roots_info`; it also accepts `JsonScanner::Selector`, but doesn't accept `JsonScanner::Options`
 
+Object keys may be duplicated in JSON. `JsonScanner.scan` returns every matching occurrence, so validate that the result has one element when a duplicate would be invalid, or use `.last` to use the usual JSON/Ruby last-key-wins behavior. `JsonScanner.parse` also retains the last matching value. It deliberately does not deduplicate overlapping selectors: duplicate keys have the same path but different source values, so path-based deduplication could retain the first value instead of the last.
+
+```ruby
+json_str = '{"a": 42, "a": 24}'
+matches = JsonScanner.scan(json_str, [["a"]]).first
+raise "duplicate key" unless matches.length == 1
+
+last_match = matches.last
+JSON.parse(json_str.byteslice(last_match[0]...last_match[1]), quirks_mode: true)
+# => 24
+
+JsonScanner.parse(json_str, [["a"]])
+# => {"a"=>24}
+```
+
+Pass a block to customize parsing selected values. For example, this enables duplicate keys in versions of the `json` gem that support `allow_duplicate_key`:
+
+```ruby
+JsonScanner.parse(json_str, [[]]) do |value_json, **options|
+  JSON.parse(value_json, **options, allow_duplicate_key: true)
+end
+# => {"a"=>24}
+```
+
 ```ruby
 JsonScanner.parse('[0, 42, 0]garbage', [[(1..-1)]], allow_trailing_garbage: true)
 # => [:stub, 42, 0]
@@ -212,7 +236,7 @@ JS
 json_with_trailing_garbage = script_text[/__APOLLO_STATE__\s*=\s*({.+)/, 1]
 json_end_pos = JsonScanner.scan(
   json_with_trailing_garbage, [[]], allow_trailing_garbage: true,
-).first.first[1]
+).first.last[1]
 apollo_state = JSON.parse(json_with_trailing_garbage[0...json_end_pos])
 ```
 

lib/json_scanner.rb

@@ -20,75 +20,115 @@ class Error < StandardError; end
   SCAN_OPTIONS = Options.new(SCAN_OPTS)
   private_constant :SCAN_OPTIONS
 
-  def self.parse(json_str, config_or_path_ary, **opts)
-    # with_path and with_roots_info is set here
-    unless (extra_opts = opts.keys - ALLOWED_OPTS).empty?
-      raise ArgumentError, "unknown keyword#{"s" if extra_opts.size > 1}: #{extra_opts.map(&:inspect).join(", ")}"
-    end
-
-    opts[:symbolize_path_keys] = opts.delete(:symbolize_names) if opts.key?(:symbolize_names)
+  def self.parse(json_str, config_or_path_ary, **opts, &parse_block)
+    symbolize_names = prepare_parse_options(opts)
     results, roots = if opts.empty?
                        scan(json_str, config_or_path_ary, SCAN_OPTIONS)
                      else
                        scan(json_str, config_or_path_ary, **opts, **SCAN_OPTS)
                      end
 
-    res = process_results(json_str, results, roots, opts[:symbolize_path_keys])
+    res = process_results(json_str, results, roots, symbolize_names, &parse_block)
 
     opts[:allow_multiple_values] ? res : res.first
   end
 
-  def self.process_results(json_str, results, roots, symbolize_names)
-    # stubs are symbols, so they can be distinguished from real values
+  def self.prepare_parse_options(opts)
+    # with_path and with_roots_info are set here
+    unless (extra_opts = opts.keys - ALLOWED_OPTS).empty?
+      message = "unknown keyword#{"s" if extra_opts.size > 1}: #{extra_opts.map(&:inspect).join(", ")}"
+      raise ArgumentError, message
+    end
+
+    opts[:symbolize_path_keys] = opts.delete(:symbolize_names) if opts.key?(:symbolize_names)
+    opts[:symbolize_path_keys]
+  end
+
+  private_class_method :prepare_parse_options
+
+  def self.process_results(json_str, results, roots, symbolize_names, &parse_block)
+    # Root type symbols can be distinguished from parsed JSON values.
     res = roots.map(&:first)
-    # results for different path matchers can overlap, in that case we will simply parse more than one time,
-    # but there shouln't be any surprises in the behavior
-    results.each do |result|
-      process_result(res, result, roots, json_str, symbolize_names)
+    # Traverse in reverse so the first matching path is the last source occurrence for duplicate JSON keys.
+    results.reverse_each do |result|
+      process_result(res, result, roots, json_str, symbolize_names, &parse_block)
     end
     res
   end
 
   private_class_method :process_results
 
-  def self.process_result(res, result, roots, json_str, symbolize_names)
-    current_root_index = 0
-    next_root = roots[1]
-    result.each do |path, (begin_pos, end_pos, _type)|
-      while next_root && begin_pos >= next_root[1]
-        current_root_index += 1
-        next_root = roots[current_root_index + 1]
-      end
+  def self.process_result(res, result, roots, json_str, symbolize_names, &parse_block)
+    current_root_index = roots.length - 1
+    result.reverse_each do |path, (begin_pos, end_pos, _type)|
+      current_root_index -= 1 while current_root_index.positive? && begin_pos < roots[current_root_index][1]
 
       # for 'res[index]' check inside insert_value
-      res[current_root_index] = nil if res[current_root_index].is_a?(Symbol)
-      insert_value(res, parse_value(json_str, begin_pos, end_pos, symbolize_names), current_root_index, path)
+      root_uninitialized = res[current_root_index].is_a?(Symbol)
+      res[current_root_index] = nil if root_uninitialized
+      insert_value(res, current_root_index, path, root_uninitialized) do
+        parse_value(json_str, begin_pos, end_pos, symbolize_names, &parse_block)
+      end
     end
   end
 
   private_class_method :process_result
 
-  def self.parse_value(json_str, begin_pos, end_pos, symbolize_names)
-    # TODO: opts for JSON.parse
-    JSON.parse(
-      json_str.byteslice(begin_pos...end_pos),
-      quirks_mode: true, symbolize_names: symbolize_names,
-    )
+  def self.parse_value(json_str, begin_pos, end_pos, symbolize_names, &parse_block)
+    value_json = json_str.byteslice(begin_pos...end_pos)
+    return JSON.parse(value_json, quirks_mode: true, symbolize_names: symbolize_names) unless parse_block
+
+    yield(value_json, quirks_mode: true, symbolize_names: symbolize_names)
   end
 
   private_class_method :parse_value
 
-  def self.insert_value(res, parsed_value, index, path)
+  def self.insert_value(res, index, path, root_uninitialized, &block)
+    if path.empty?
+      return assign_value(res, index, &block) if root_uninitialized
+
+      return
+    end
+
+    res, index = descend_to_value(res, index, path)
+    return if value_present?(res, index)
+
+    fill_array_index(res, index)
+    assign_value(res, index, &block)
+  end
+
+  private_class_method :insert_value
+
+  def self.descend_to_value(res, index, path)
     until path.empty?
       new_index = path.shift
+      fill_array_index(res, index)
       res[index] ||= new_index.is_a?(Integer) ? [] : {}
       res = res[index]
       index = new_index
     end
+    [res, index]
+  end
+
+  private_class_method :descend_to_value
 
-    (index - res.size).times { res.push(STUB) } if res.is_a?(Array) && res.size < index
-    res[index] = parsed_value
+  def self.value_present?(res, index)
+    res.is_a?(Hash) ? res.key?(index) : res.size > index && res[index] != STUB
   end
 
-  private_class_method :insert_value
+  private_class_method :value_present?
+
+  def self.assign_value(res, index)
+    res[index] = yield
+  end
+
+  private_class_method :assign_value
+
+  def self.fill_array_index(res, index)
+    return unless res.is_a?(Array) && res.size < index
+
+    (index - res.size).times { res.push(STUB) }
+  end
+
+  private_class_method :fill_array_index
 end

spec/json_scanner_spec.rb

@@ -14,6 +14,9 @@
       result = described_class.scan('["1", {"a": 2}]', [[0], [1, "a"], []])
       expect(result).to eq([[[1, 4, :string]], [[12, 13, :number]], [[0, 15, :array]]])
       expect(described_class.scan('"2"', [[]])).to eq([[[0, 3, :string]]])
+      expect(described_class.scan('{"a": 42, "a": 24}', [["a"]])).to eq(
+        [[[6, 8, :number], [15, 17, :number]]],
+      )
       expect(
         described_class.scan("[0,1,2,3,4,5,6,7]", [[(0..2)], [(4...6)]]),
       ).to eq(
@@ -470,6 +473,36 @@
     it "handles overlapping selectors" do
       expect(described_class.parse('{"a": 42}', [["a"], [described_class::ANY_KEY]])).to eq({ "a" => 42 })
     end
+
+    it "retains the last value for duplicate keys" do
+      expect(described_class.parse('{"a": 42, "a": 24}', [["a"]])).to eq({ "a" => 24 })
+    end
+
+    it "allows custom value parsing" do
+      expect(described_class.parse("42", [[]]) { :parsed }).to eq(:parsed)
+    end
+
+    if JSON::VERSION >= "2"
+      it "allows custom value parsing to enable duplicate keys" do
+        value = described_class.parse('{"a": 42, "a": 24}', [[]]) do |value_json, **options|
+          JSON.parse(value_json, **options, allow_duplicate_key: true)
+        end
+
+        expect(value).to eq({ "a" => 24 })
+      end
+    end
+
+    it "parses overlapping selectors only once" do
+      calls = 0
+
+      value = described_class.parse('{"a": 42}', [["a"], [described_class::ANY_KEY]]) do |value_json, **options|
+        calls += 1
+        JSON.parse(value_json, **options)
+      end
+
+      expect(value).to eq({ "a" => 42 })
+      expect(calls).to eq(1)
+    end
   end
 
   describe described_class::Selector do