fix CI errors

Signed-off-by: Varsha U N <[email protected]>

Author: VarshaUN

scancodeio/settings.py

@@ -23,12 +23,13 @@
 import sys
 import tempfile
 from pathlib import Path
-from venv import logger
+import logging
 
 import environ
 
 from scanpipe.archiving import LocalFilesystemProvider
 
+
 PROJECT_DIR = environ.Path(__file__) - 1
 ROOT_DIR = PROJECT_DIR - 1
 
@@ -376,9 +377,10 @@
 
 CRISPY_TEMPLATE_PACK = "bootstrap3"
 
-# Storing archives locally (Package Storage settings)
-
-ENABLE_DOWNLOAD_ARCHIVING = env.bool("ENABLE_DOWNLOAD_ARCHIVING", default=False)
+# Centralized archive directory for all projects
+CENTRAL_ARCHIVE_PATH = env.str(
+    "CENTRAL_ARCHIVE_PATH", default="/var/scancodeio/archives"
+)
 
 # localstorage configuration
 DOWNLOAD_ARCHIVING_PROVIDER = env.str(
@@ -393,15 +395,15 @@
 # Initialize the DownloadStore for local storage
 
 download_store = None
-if ENABLE_DOWNLOAD_ARCHIVING:
-    if DOWNLOAD_ARCHIVING_PROVIDER == "localstorage":
-        config = DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION or {}
-        root_path = Path(config.get("root_path", "/var/scancodeio/downloads"))
-        try:
-            download_store = LocalFilesystemProvider(root_path=root_path)
-        except Exception as e:
-            logger.error(f"Failed to initialize LocalFilesystemProvider: {e}")
-    else:
+logger = logging.getLogger(__name__)
+if DOWNLOAD_ARCHIVING_PROVIDER == "localstorage":
+    config = DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION or {}
+    root_path = Path(config.get("root_path", CENTRAL_ARCHIVE_PATH))
+    try:
+        download_store = LocalFilesystemProvider(root_path=root_path)
+    except Exception as e:
+        logger.error(f"Failed to initialize LocalFilesystemProvider: {e}")
+else:
         logger.error(
             f"Unknown DOWNLOAD_ARCHIVING_PROVIDER: {DOWNLOAD_ARCHIVING_PROVIDER}"
         )

scanpipe/archiving.py

@@ -63,7 +63,7 @@ def _build_metadata(
             "sha256": sha256,
             "filename": filename,
             "download_date": download_date,
-            "url": download_url,
+            "download_url": download_url,
         }
 
     @abstractmethod

scanpipe/pipelines/__init__.py

@@ -23,6 +23,7 @@
 import inspect
 import logging
 import traceback
+import hashlib
 from contextlib import contextmanager
 from datetime import datetime
 from functools import wraps
@@ -34,7 +35,6 @@
 from pyinstrument import Profiler
 
 from aboutcode.pipeline import BasePipeline
-from scancodeio.settings import ENABLE_DOWNLOAD_ARCHIVING
 from scancodeio.settings import download_store
 
 logger = logging.getLogger(__name__)
@@ -148,9 +148,24 @@ def download_missing_inputs(self):
                 error_tracebacks.append((msg, "No traceback available."))
                 continue
 
+            download_url = input_source.download_url
+            if not download_url:
+                continue
+
+            url_hash = hashlib.sha256(download_url.encode()).hexdigest()
+            filename = input_source.filename or Path(download_url).name or f"{url_hash}.archive"
+            archive_path = Path(settings.CENTRAL_ARCHIVE_PATH) / url_hash / filename
+
+            if archive_path.exists():
+                logger.info(f"Reusing existing archive at {archive_path}")
+                input_source.file_path = str(archive_path)
+                input_source.save()
+                continue
+
             self.log(f"Fetching input from {input_source.download_url}")
             try:
                 input_source.fetch()
+                
             except Exception as error:
                 traceback_str = traceback.format_exc()
                 logger.error(traceback_str)

scanpipe/pipes/input.py

@@ -44,7 +44,6 @@
 from scanpipe.models import InputSource
 from scanpipe.pipes import scancode
 from scanpipe.pipes.output import mappings_key_by_fieldname
-from scancodeio.settings import ENABLE_DOWNLOAD_ARCHIVING
 from scancodeio.settings import download_store
 
 logger = logging.getLogger(__name__)
@@ -262,61 +261,47 @@ def add_input_from_url(project, url, filename=None):
         logger.error(f"Failed to download {url}: {e}")
         raise
 
-    should_archive = (
-        ENABLE_DOWNLOAD_ARCHIVING == "always"
-        or (
-            ENABLE_DOWNLOAD_ARCHIVING == "per_project"
-            and getattr(project, "archive_downloads", False)
-        )
-        or (
-            ENABLE_DOWNLOAD_ARCHIVING == "per_input"
-            and "archive" in getattr(project, "input_tags", [])
-        )
-    )
+    filename = filename or url.split("/")[-1] or "downloaded_file"
+    url_hash = hashlib.sha256(url.encode()).hexdigest()
+    archive_path = Path(project.settings.CENTRAL_ARCHIVE_PATH) / url_hash / filename
 
-    filename = filename or url.split("/")[-1]
-    if should_archive and download_store:
-        sha256 = hashlib.sha256(content).hexdigest()
-        existing_download = download_store.get(sha256)
-        if not existing_download:
-            try:
-                download = download_store.put(
-                    content=content,
-                    download_url=url,
-                    download_date=datetime.now().isoformat(),
-                    filename=filename,
-                )
-            except Exception as e:
-                logger.error(f"Failed to archive download for {url}: {e}")
-                raise
-        else:
-            download = existing_download
-
-        InputSource.objects.create(
-            project=project,
-            sha256=download.sha256,
-            download_url=download.download_url,
-            filename=download.filename,
-            download_date=download.download_date,
-            is_uploaded=False,
-        )
+    if download_store:
+        try:
+            download = download_store.put(
+                content=content,
+                download_url=url,
+                download_date=datetime.now().isoformat(),
+                filename=filename,
+            )
+            InputSource.objects.create(
+                project=project,
+                sha256=download.sha256,
+                download_url=download.download_url,
+                filename=download.filename,
+                download_date=download.download_date,
+                file_path=str(download.path),
+                is_uploaded=False,
+            )
+        except Exception as e:
+            logger.error(f"Failed to archive download for {url}: {e}")
+            raise
     else:
         input_path = project.input_path / filename
         try:
+            input_path.parent.mkdir(parents=True, exist_ok=True)
             with open(input_path, "wb") as f:
                 f.write(content)
+            InputSource.objects.create(
+                project=project,
+                filename=filename,
+                download_url=url,
+                file_path=str(input_path),
+                is_uploaded=False,
+            )
         except Exception as e:
             logger.error(f"Failed to save {filename} to {input_path}: {e}")
             raise
 
-        InputSource.objects.create(
-            project=project,
-            filename=filename,
-            download_url=url,
-            is_uploaded=False,
-        )
-
-
 def add_input_from_upload(project, uploaded_file):
     """
     Add an uploaded file as an InputSource for the specified ``project``.
@@ -325,54 +310,38 @@ def add_input_from_upload(project, uploaded_file):
     content = uploaded_file.read()
     filename = uploaded_file.name
 
-    should_archive = (
-        ENABLE_DOWNLOAD_ARCHIVING == "always"
-        or (
-            ENABLE_DOWNLOAD_ARCHIVING == "per_project"
-            and getattr(project, "archive_downloads", False)
-        )
-        or (
-            ENABLE_DOWNLOAD_ARCHIVING == "per_input"
-            and "archive" in getattr(project, "input_tags", [])
-        )
-    )
-
-    if should_archive and download_store:
-        sha256 = hashlib.sha256(content).hexdigest()
-        existing_download = download_store.get(sha256)
-        if not existing_download:
-            try:
-                download = download_store.put(
-                    content=content,
-                    download_url="",  # No URL for uploads
-                    download_date=datetime.now().isoformat(),
-                    filename=filename,
-                )
-            except Exception as e:
-                logger.error(f"Failed to archive upload {filename}: {e}")
-                raise
-        else:
-            download = existing_download
-
-        InputSource.objects.create(
-            project=project,
-            sha256=download.sha256,
-            download_url=download.download_url,
-            filename=download.filename,
-            download_date=download.download_date,
-            is_uploaded=True,
-        )
+    if download_store:
+        try:
+            download = download_store.put(
+                content=content,
+                download_url="",
+                download_date=datetime.now().isoformat(),
+                filename=filename,
+            )
+            InputSource.objects.create(
+                project=project,
+                sha256=download.sha256,
+                download_url=download.download_url,
+                filename=download.filename,
+                download_date=download.download_date,
+                file_path=str(download.path),
+                is_uploaded=True,
+            )
+        except Exception as e:
+            logger.error(f"Failed to archive upload {filename}: {e}")
+            raise
     else:
         input_path = project.input_path / filename
         try:
+            input_path.parent.mkdir(parents=True, exist_ok=True)
             with open(input_path, "wb") as f:
                 f.write(content)
+            InputSource.objects.create(
+                project=project,
+                filename=filename,
+                file_path=str(input_path),
+                is_uploaded=True,
+            )
         except Exception as e:
             logger.error(f"Failed to save {filename} to {input_path}: {e}")
-            raise
-
-        InputSource.objects.create(
-            project=project,
-            filename=filename,
-            is_uploaded=True,
-        )
+            raise