When reaching the end of a chunk while scanning for differences, look ahead into the next chunk to detect whether the next value is an object/array or not.

Author: nicktobey

go/store/prolly/tree/indexed_json_diff.go

@@ -245,10 +245,24 @@ func (jd *IndexedJsonDiffer) Next(ctx context.Context) (diff JsonDiff, err error
 			case 0:
 				key := fromCurrentLocation.Clone().key
 
+				fromNextCharacter, err := jd.currentFromCursor.nextCharacter(ctx)
+				if err == io.EOF {
+					return JsonDiff{}, jsonParseError
+				}
+				if err != nil {
+					return JsonDiff{}, err
+				}
+				toNextCharacter, err := jd.currentToCursor.nextCharacter(ctx)
+				if err == io.EOF {
+					return JsonDiff{}, jsonParseError
+				}
+				if err != nil {
+					return JsonDiff{}, err
+				}
 				// Both sides have the same key. If they're both an object or both an array, continue.
 				// Otherwise, compare them and possibly return a modification.
-				if (fromScanner.current() == '{' && toScanner.current() == '{') ||
-					(fromScanner.current() == '[' && toScanner.current() == '[') {
+				if (fromNextCharacter == '{' && toNextCharacter == '{') ||
+					(fromNextCharacter == '[' && toNextCharacter == '[') {
 					err = advanceCursor(ctx, &jd.currentFromCursor)
 					if err != nil {
 						return JsonDiff{}, err

go/store/prolly/tree/json_cursor.go

@@ -195,20 +195,28 @@ func (j *JsonCursor) AdvanceToLocation(ctx context.Context, path jsonLocation, f
 	return true, nil
 }
 
+func (j *JsonCursor) advanceCursor(ctx context.Context) error {
+	err := j.cur.advance(ctx)
+	if err != nil {
+		return err
+	}
+	if !j.cur.Valid() {
+		// We hit the end of the tree. This shouldn't happen.
+		return io.EOF
+	}
+	j.jsonScanner = ScanJsonFromMiddle(j.cur.currentValue(), j.jsonScanner.currentPath)
+	return nil
+}
+
 func (j *JsonCursor) AdvanceToNextLocation(ctx context.Context) (crossedBoundary bool, err error) {
 	err = j.jsonScanner.AdvanceToNextLocation()
 	if err == io.EOF {
 		crossedBoundary = true
 		// We hit the end of the chunk, load the next one
-		err = j.cur.advance(ctx)
+		err = j.advanceCursor(ctx)
 		if err != nil {
-			return
-		}
-		if !j.cur.Valid() {
-			// We hit the end of the tree. This shouldn't happen.
-			return true, io.EOF
+			return false, err
 		}
-		j.jsonScanner = ScanJsonFromMiddle(j.cur.currentValue(), j.jsonScanner.currentPath)
 		return true, j.jsonScanner.AdvanceToNextLocation()
 	} else if err != nil {
 		return
@@ -221,6 +229,12 @@ func (j *JsonCursor) GetCurrentPath() jsonLocation {
 	return j.jsonScanner.currentPath
 }
 
-func (j *JsonCursor) nextCharacter() byte {
-	return j.jsonScanner.jsonBuffer[j.jsonScanner.valueOffset]
+func (j *JsonCursor) nextCharacter(ctx context.Context) (byte, error) {
+	if j.jsonScanner.atEndOfChunk() {
+		err := j.advanceCursor(ctx)
+		if err != nil {
+			return 255, err
+		}
+	}
+	return j.jsonScanner.jsonBuffer[j.jsonScanner.valueOffset], nil
 }