NRL-1215 - Revert "NRL-1215 Enhance duplicate detection for JSON arrays and add corresponding tests"

This reverts commit 3d43af2.

Author: axelkrastek1-nhs

layer/nrlf/core/json_duplicate_checker.py

@@ -25,8 +25,6 @@ def __init__(self):
         # Track keys at each path level to detect duplicates
         self.key_registry: Dict[str, Dict[str, bool]] = {}
         self.current_duplicate_index: Dict[str, int] = {}
-        # Track seen array elements to detect duplicates
-        self.seen_array_elements: Dict[str, List[JsonValue]] = {}
 
     def get_path_with_index(self, path: List[str], key: str) -> List[str]:
         current_level = ".".join(path)
@@ -78,26 +76,8 @@ def traverse_array(self, items: JsonArray, path: list[str]) -> None:
         """Process JSON array items while updating the path for duplicates."""
         array_path = path[-1]
         base_path = path[:-1]
-        seen_elements = self.seen_array_elements.setdefault(".".join(path), set())
 
         for idx, item in enumerate(items):
-            serialized_item = json.dumps(item, sort_keys=True)
-            if serialized_item in seen_elements:
-                element = f"{array_path}[{idx}]"
-                duplicate_path = ".".join(base_path + [element])
-                self.duplicate_keys_and_paths.setdefault(element, []).append(
-                    duplicate_path
-                )
-                print(f"Found duplicate array element at path: {duplicate_path}")
-            else:
-                seen_elements.add(serialized_item)
-
-            # if item in seen_elements:
-            #     duplicate_path = f"{array_path}[{idx}]"
-            #     self.duplicate_keys_and_paths.setdefault(duplicate_path, []).append(f"{base_path[0]}.{duplicate_path}")
-            #     print(f"Found duplicate array element at path: {duplicate_path}")
-            # else:
-            #     seen_elements.append(item)
             if not isinstance(item, (list, tuple)):
                 continue
             self.process_collection(item, base_path, f"{array_path}[{idx}]")

layer/nrlf/core/tests/test_json_duplicate_checker.py

@@ -324,60 +324,3 @@ def test_array_edge_case_duplicate(self):
         duplicates, paths = check_duplicate_keys(json_content)
         self.assertEqual(duplicates, ["array"])
         self.assertEqual(paths, ["root.array"])
-
-    def test_array_element_duplicate(self):
-        json_content = """
-        {
-            "array": [
-                1,
-                2,
-                3,
-                1
-            ]
-        }
-        """
-        duplicates, paths = check_duplicate_keys(json_content)
-        self.assertEqual(duplicates, ["array[3]"])
-        self.assertEqual(paths, ["root.array[3]"])
-
-    # deeply nested object with a deeply nested array with a duplicate
-    def test_deeply_nested_object_with_deeply_nested_array_duplicate(self):
-        json_content = """
-        {
-            "root": {
-                "level1": {
-                    "level2": {
-                        "level3": {
-                            "level4": {
-                                "level5": {
-                                    "level6": {
-                                        "level7": {
-                                            "level8": {
-                                                "level9": {
-                                                    "level10": {
-                                                        "array": [
-                                                            {"key1": 1, "key2": 2},
-                                                            {"key1": 1, "key2": 2}
-                                                        ]
-                                                    }
-                                                }
-                                            }
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        """
-        duplicates, paths = check_duplicate_keys(json_content)
-        self.assertEqual(duplicates, ["array[1]"])
-        # duplicate root here needs fixing in traverse_array loop
-        self.assertEqual(
-            paths,
-            [
-                "root.root.level1.level2.level3.level4.level5.level6.level7.level8.level9.level10.array[1]"
-            ],
-        )