pass endpoint not url to pipeline

Author: pooleycodes

request-processor/src/application/core/pipeline.py

@@ -195,7 +195,7 @@ def fetch_add_data_response(
     output_path,
     specification_dir,
     cache_dir,
-    url,
+    endpoint,
 ):
     try:
         specification = Specification(specification_dir)
@@ -228,7 +228,7 @@ def fetch_add_data_response(
                     resource=resource_from_path(resource_file_path),
                     valid_category_values=valid_category_values,
                     disable_lookups=False,
-                    endpoints=[url],
+                    endpoints=[endpoint],
                 )
 
                 existing_entities.extend(
@@ -254,7 +254,7 @@ def fetch_add_data_response(
                         pipeline_dir=pipeline_dir,
                         specification=specification,
                         cache_dir=cache_dir,
-                        endpoints=[url] if url else None,
+                        endpoints=[endpoint] if endpoint else None,
                     )
                     logger.info(
                         f"Found {len(new_lookups)} unidentified lookups in {resource_file}"
@@ -281,7 +281,7 @@ def fetch_add_data_response(
                         resource=resource_from_path(resource_file_path),
                         valid_category_values=valid_category_values,
                         disable_lookups=False,
-                        endpoints=[url],
+                        endpoints=[endpoint],
                     )
                 else:
                     logger.info(f"No unidentified lookups found in {resource_file}")

request-processor/src/application/core/workflow.py

@@ -1,4 +1,5 @@
 import datetime
+import hashlib
 import os
 import csv
 from pathlib import Path
@@ -463,6 +464,8 @@ def add_data_workflow(
             ] = f"Unable to find lookups for collection '{collection}', dataset '{dataset}'"
             return response_data
 
+        endpoint_hash = hashlib.sha256(url.encode("utf-8")).hexdigest()
+
         # All processes arount transforming the data and generating pipeline summary
         pipeline_summary = fetch_add_data_response(
             dataset=dataset,
@@ -472,7 +475,7 @@ def add_data_workflow(
             output_path=output_path,
             specification_dir=directories.SPECIFICATION_DIR,
             cache_dir=directories.CACHE_DIR,
-            url=url,
+            endpoint=endpoint_hash,
         )
 
         # Create endpoint and source summaries in workflow

request-processor/tests/unit/src/application/core/test_pipeline.py

@@ -17,7 +17,7 @@ def test_fetch_add_data_response_success(monkeypatch, tmp_path):
     input_path = tmp_path / "resource"
     specification_dir = tmp_path / "specification"
     cache_dir = tmp_path / "cache"
-    url = "http://example.com/endpoint"
+    endpoint = "abc123hash"
 
     input_path.mkdir(parents=True)
     pipeline_dir.mkdir(parents=True)
@@ -68,7 +68,7 @@ def test_fetch_add_data_response_success(monkeypatch, tmp_path):
         output_path=str(input_path / "output.csv"),
         specification_dir=str(specification_dir),
         cache_dir=str(cache_dir),
-        url=url,
+        endpoint=endpoint,
     )
 
     assert "new-in-resource" in result
@@ -83,7 +83,7 @@ def test_fetch_add_data_response_no_files(monkeypatch, tmp_path):
     input_path = tmp_path / "resource"
     specification_dir = tmp_path / "specification"
     cache_dir = tmp_path / "cache"
-    url = "http://example.com/endpoint"
+    endpoint = "abc123hash"
 
     input_path.mkdir(parents=True)
     pipeline_dir.mkdir(parents=True)
@@ -103,7 +103,7 @@ def test_fetch_add_data_response_no_files(monkeypatch, tmp_path):
         output_path=str(input_path / "output.csv"),
         specification_dir=str(specification_dir),
         cache_dir=str(cache_dir),
-        url=url,
+        endpoint=endpoint,
     )
 
     assert "new-in-resource" in result
@@ -118,7 +118,7 @@ def test_fetch_add_data_response_file_not_found(monkeypatch, tmp_path):
     input_path = tmp_path / "nonexistent"
     specification_dir = tmp_path / "specification"
     cache_dir = tmp_path / "cache"
-    url = "http://example.com/endpoint"
+    endpoint = "abc123hash"
 
     pipeline_dir.mkdir(parents=True)
 
@@ -138,7 +138,7 @@ def test_fetch_add_data_response_file_not_found(monkeypatch, tmp_path):
             output_path=str(input_path / "output.csv"),
             specification_dir=str(specification_dir),
             cache_dir=str(cache_dir),
-            url=url,
+            endpoint=endpoint,
         )
 
 
@@ -150,7 +150,7 @@ def test_fetch_add_data_response_handles_processing_error(monkeypatch, tmp_path)
     input_path = tmp_path / "resource"
     specification_dir = tmp_path / "specification"
     cache_dir = tmp_path / "cache"
-    url = "http://example.com/endpoint"
+    endpoint = "abc123hash"
 
     input_path.mkdir(parents=True)
     pipeline_dir.mkdir(parents=True)
@@ -178,7 +178,7 @@ def raise_exception(*args, **kwargs):
         output_path=str(input_path / "output.csv"),
         specification_dir=str(specification_dir),
         cache_dir=str(cache_dir),
-        url=url,
+        endpoint=endpoint,
     )
 
     assert "new-in-resource" in result

request-processor/tests/unit/src/application/core/test_workflow.py

@@ -9,6 +9,7 @@
     fetch_add_data_collection_csvs,
 )
 import csv
+import hashlib
 import os
 from pathlib import Path
 import urllib
@@ -409,7 +410,7 @@ def fake_fetch_add_data_response(
         output_path,
         specification_dir,
         cache_dir,
-        url,
+        endpoint,
     ):
         called["fetch_add_data_response"] = {
             "dataset": dataset,
@@ -419,7 +420,7 @@ def fake_fetch_add_data_response(
             "output_path": output_path,
             "specification_dir": specification_dir,
             "cache_dir": cache_dir,
-            "url": url,
+            "endpoint": endpoint,
         }
         return {"result": "ok"}
 
@@ -470,7 +471,8 @@ def fake_fetch_add_data_response(
         == directories.SPECIFICATION_DIR
     )
     assert called["fetch_add_data_response"]["cache_dir"] == directories.CACHE_DIR
-    assert called["fetch_add_data_response"]["url"] == url
+    expected_endpoint_hash = hashlib.sha256(url.encode("utf-8")).hexdigest()
+    assert called["fetch_add_data_response"]["endpoint"] == expected_endpoint_hash
 
 
 def test_fetch_add_data_pipeline_csvs_from_url(monkeypatch, tmp_path):