feat: add referer header to download request (#1366)

Author: davidgamez

functions-python/batch_process_dataset/src/main.py

@@ -332,8 +332,9 @@ def generate_temp_filename(self):
         """
         Generates a temporary filename
         """
+        working_dir = os.getenv("WORKING_DIR", "/in-memory")
         temporary_file_path = (
-            f"/in-memory/{self.feed_stable_id}-{random.randint(0, 1000000)}.zip"
+            f"{working_dir}/{self.feed_stable_id}-{random.randint(0, 1000000)}.zip"
         )
         return temporary_file_path
 

functions-python/batch_process_dataset/src/scripts/download_verifier.py

@@ -68,6 +68,10 @@ def verify_upload_dataset(producer_url: str):
     # Replace with actual producer URL
     try:
         os.environ["STORAGE_EMULATOR_HOST"] = f"http://{HOST}:{PORT}"
+        os.environ["WORKING_DIR"] = "/tmp/verifier"
+        # create working dir if not exists
+        if not os.path.exists(os.environ["WORKING_DIR"]):
+            os.makedirs(os.environ["WORKING_DIR"])
         server = create_server(
             host=HOST, port=PORT, in_memory=False, default_bucket=BUCKET_NAME
         )

functions-python/helpers/tests/test_helpers.py

@@ -104,6 +104,7 @@ def test_download_and_get_hash_auth_type_header(self):
                 preload_content=False,
                 headers={
                     "User-Agent": expected_user_agent,
+                    "Referer": url,
                     api_key_parameter_name: credentials,
                 },
                 redirect=True,
@@ -155,7 +156,7 @@ def test_download_and_get_hash_auth_type_api_key(self):
                 "GET",
                 modified_url,
                 preload_content=False,
-                headers={"User-Agent": expected_user_agent},
+                headers={"User-Agent": expected_user_agent, "Referer": base_url},
                 redirect=True,
             )
 

functions-python/helpers/utils.py

@@ -149,7 +149,8 @@ def download_and_get_hash(
         headers = {
             "User-Agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) "
             "AppleWebKit/537.36 (KHTML, like Gecko) "
-            "Chrome/126.0.0.0 Mobile Safari/537.36"
+            "Chrome/126.0.0.0 Mobile Safari/537.36",
+            "Referer": url,
         }
         # Careful, some URLs may already contain a query string
         # (e.g. http://api.511.org/transit/datafeeds?operator_id=CE)