NRL-1215 Enhance duplicate detection for JSON arrays and add corresponding tests

Author: axelkrastek1-nhs

layer/nrlf/core/json_duplicate_checker.py

@@ -25,6 +25,8 @@ def __init__(self):
         # Track keys at each path level to detect duplicates
         self.key_registry: Dict[str, Dict[str, bool]] = {}
         self.current_duplicate_index: Dict[str, int] = {}
+        # Track seen array elements to detect duplicates
+        self.seen_array_elements: Dict[str, List[JsonValue]] = {}
 
     def get_path_with_index(self, path: List[str], key: str) -> List[str]:
         current_level = ".".join(path)
@@ -76,8 +78,24 @@ def traverse_array(self, items: JsonArray, path: list[str]) -> None:
         """Process JSON array items while updating the path for duplicates."""
         array_path = path[-1]
         base_path = path[:-1]
+        seen_elements = self.seen_array_elements.setdefault(".".join(path), set())
 
         for idx, item in enumerate(items):
+            serialized_item = json.dumps(item, sort_keys=True)
+            if serialized_item in seen_elements:
+                element = f"{array_path}[{idx}]"
+                duplicate_path = ".".join(base_path + [element])
+                self.duplicate_keys_and_paths.setdefault(element, []).append(duplicate_path)
+                print(f"Found duplicate array element at path: {duplicate_path}")
+            else:
+                seen_elements.add(serialized_item)
+
+            # if item in seen_elements:
+            #     duplicate_path = f"{array_path}[{idx}]"
+            #     self.duplicate_keys_and_paths.setdefault(duplicate_path, []).append(f"{base_path[0]}.{duplicate_path}")
+            #     print(f"Found duplicate array element at path: {duplicate_path}")
+            # else:
+            #     seen_elements.append(item)
             if not isinstance(item, (list, tuple)):
                 continue
             self.process_collection(item, base_path, f"{array_path}[{idx}]")

layer/nrlf/core/tests/test_json_duplicate_checker.py

@@ -323,4 +323,56 @@ def test_array_edge_case_duplicate(self):
         """
         duplicates, paths = check_duplicate_keys(json_content)
         self.assertEqual(duplicates, ["array"])
-        self.assertEqual(paths, ["root.array"])
+        self.assertEqual(paths, ["root.array"])
+
+    def test_array_element_duplicate(self):
+        json_content = """
+        {
+            "array": [
+                1,
+                2,
+                3,
+                1
+            ]
+        }
+        """
+        duplicates, paths = check_duplicate_keys(json_content)
+        self.assertEqual(duplicates, ["array[3]"])
+        self.assertEqual(paths, ["root.array[3]"])
+
+    # deeply nested object with a deeply nested array with a duplicate
+    def test_deeply_nested_object_with_deeply_nested_array_duplicate(self):
+        json_content = """
+        {
+            "root": {
+                "level1": {
+                    "level2": {
+                        "level3": {
+                            "level4": {
+                                "level5": {
+                                    "level6": {
+                                        "level7": {
+                                            "level8": {
+                                                "level9": {
+                                                    "level10": {
+                                                        "array": [
+                                                            {"key1": 1, "key2": 2},
+                                                            {"key1": 1, "key2": 2}
+                                                        ]
+                                                    }
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        """
+        duplicates, paths = check_duplicate_keys(json_content)
+        self.assertEqual(duplicates, ["array[1]"])
+        # duplicate root here needs fixing in traverse_array loop
+        self.assertEqual(paths, ["root.root.level1.level2.level3.level4.level5.level6.level7.level8.level9.level10.array[1]"])