Merge pull request #350 from digital-land/referential-check-issue-type

Update lookup

Author: kena-SL

digital_land/command_arguments.py

@@ -103,5 +103,5 @@ def provision_summary_dir(f):
     return click.option(
         "--provision-summary-dir",
         type=click.Path(exists=True),
-        default="var/cache/provision-summary/",
+        default="var/cache/provision-summary.csv",
     )(f)

digital_land/commands.py

@@ -216,7 +216,7 @@ def pipeline_run(
     resource=None,
     output_log_dir=None,
     converted_path=None,
-    provision_summary_dir="var/cache/provision_summary",
+    provision_summary_dir="var/cache/provision_summary.csv",
 ):
     # set up paths
     cache_dir = Path(cache_dir)

digital_land/phase/lookup.py

@@ -174,16 +174,15 @@ def process(self, stream):
                             row.get(linked_dataset, "")
                             or row.get(linked_dataset, "").strip()
                         ):
-                            get_organisations = pd.read_csv(
-                                self.provision_summary_dir
-                                + "/"
-                                + linked_dataset
-                                + ".csv"
-                            )
+                            get_organisations = pd.read_csv(self.provision_summary_dir)
+
+                            dataset_org = get_organisations[
+                                get_organisations["dataset"] == linked_dataset
+                            ]
 
                             if (
                                 row.get("organisation", "")
-                                in get_organisations["organisation"].values
+                                in dataset_org["organisation"].values
                             ):
                                 reference = row.get(linked_dataset, "")
                                 find_entity = self.lookup(

tests/unit/phase/test_lookup.py

@@ -157,7 +157,12 @@ def test_no_associated_documents_issue(
             provision_summary_dir="var/cache/provision-summary/",
         )
         phase.entity_field = "entity"
-        mock_df = pd.DataFrame({"organisation": ["local-authority:ABC"]})
+        mock_df = pd.DataFrame(
+            {
+                "organisation": ["local-authority:ABC"],
+                "dataset": ["article-4-direction"],
+            }
+        )
         mocker.patch("pandas.read_csv", return_value=mock_df)
         output = [block for block in phase.process(input_stream)]
 
@@ -185,7 +190,12 @@ def test_no_associated_documents_issue_for_missing_dataset(
             provision_summary_dir="var/cache/provision-summary/",
         )
         phase.entity_field = "entity"
-        mock_df = pd.DataFrame({"organisation": ["local-authority:XYZ"]})
+        mock_df = pd.DataFrame(
+            {
+                "organisation": ["local-authority:XYZ", "local-authority:ABC"],
+                "dataset": ["article-4-direction", "article-4-direction-area"],
+            }
+        )
         mocker.patch("pandas.read_csv", return_value=mock_df)
         output = [block for block in phase.process(input_stream)]
 
@@ -211,7 +221,12 @@ def test_no_associated_documents_issue_for_retired_entity(
             provision_summary_dir="var/cache/provision-summary/",
         )
         phase.entity_field = "entity"
-        mock_df = pd.DataFrame({"organisation": ["local-authority:ABC"]})
+        mock_df = pd.DataFrame(
+            {
+                "organisation": ["local-authority:ABC"],
+                "dataset": ["article-4-direction"],
+            }
+        )
         mocker.patch("pandas.read_csv", return_value=mock_df)
         output = [block for block in phase.process(input_stream)]