Add download archiving system with LocalFilesystem, S3, and SFTP providers

Signed-off-by: Varsha U N <[email protected]>

Author: VarshaUN

scancodeio/settings.py

@@ -23,8 +23,10 @@
 import sys
 import tempfile
 from pathlib import Path
-
+from venv import logger
 import environ
+from scanpipe.archiving import LocalFilesystemProvider, S3LikeProvider, SftpProvider
+
 
 PROJECT_DIR = environ.Path(__file__) - 1
 ROOT_DIR = PROJECT_DIR - 1
@@ -371,6 +373,59 @@
 
 CRISPY_TEMPLATE_PACK = "bootstrap3"
 
+# Storing archives locally or in S3 (Package Storage settings)
+
+ENABLE_DOWNLOAD_ARCHIVING = env.bool("ENABLE_DOWNLOAD_ARCHIVING", default=False)
+
+# localstorage, s3, sftp
+DOWNLOAD_ARCHIVING_PROVIDER = env.str("DOWNLOAD_ARCHIVING_PROVIDER", default="localstorage")
+
+# For local storage, we would store the root path in that setting
+DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION = env.dict("DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION", default=None)
+
+# Initialize the DownloadStore based on provider
+
+download_store = None
+if ENABLE_DOWNLOAD_ARCHIVING:
+    if DOWNLOAD_ARCHIVING_PROVIDER == "localstorage":
+        config = DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION or {}
+        root_path = Path(config.get("root_path", "/var/scancodeio/downloads"))
+        try:
+            download_store = LocalFilesystemProvider(root_path=root_path)
+        except Exception as e:
+            logger.error(f"Failed to initialize LocalFilesystemProvider: {e}")
+    elif DOWNLOAD_ARCHIVING_PROVIDER == "s3":
+        config = DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION or {}
+        required_keys = ["bucket_name", "aws_userid", "aws_apikey"]
+        if not all(key in config for key in required_keys):
+            logger.error(f"S3 provider requires {required_keys} in DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION")
+        else:
+            try:
+                download_store = S3LikeProvider(
+                    bucket_name=config.get("bucket_name"),
+                    aws_userid=config.get("aws_userid"),
+                    aws_apikey=config.get("aws_apikey"),
+                    other_aws_credentials=config.get("other_aws_credentials", {}),
+                )
+            except Exception as e:
+                logger.error(f"Failed to initialize S3LikeProvider: {e}")
+    elif DOWNLOAD_ARCHIVING_PROVIDER == "sftp":
+        config = DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION or {}
+        required_keys = ["host", "root_path", "ssh_credentials"]
+        if not all(key in config for key in required_keys):
+            logger.error(f"SFTP provider requires {required_keys} in DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION")
+        else:
+            try:
+                download_store = SftpProvider(
+                    host=config.get("host"),
+                    root_path=config.get("root_path"),
+                    ssh_credentials=config.get("ssh_credentials", {}),
+                )
+            except Exception as e:
+                logger.error(f"Failed to initialize SftpProvider: {e}")
+    else:
+        logger.error(f"Unknown DOWNLOAD_ARCHIVING_PROVIDER: {DOWNLOAD_ARCHIVING_PROVIDER}")
+
 # Job Queue
 
 RQ_QUEUES = {