modify the required imports

Signed-off-by: Varsha U N <[email protected]>

Author: VarshaUN

Dockerfile

@@ -91,4 +91,4 @@ COPY --chown=$APP_USER:$APP_USER pyproject.toml $APP_DIR/
 RUN pip install --no-cache-dir .
 
 # Copy the codebase and set the proper permissions for the APP_USER
-COPY --chown=$APP_USER:$APP_USER . $APP_DIR
+COPY --chown=$APP_USER:$APP_USER . $APP_DIR

scancodeio/settings.py

@@ -28,8 +28,6 @@
 import environ
 
 from scanpipe.archiving import LocalFilesystemProvider
-from scanpipe.archiving import S3LikeProvider
-from scanpipe.archiving import SftpProvider
 
 PROJECT_DIR = environ.Path(__file__) - 1
 ROOT_DIR = PROJECT_DIR - 1
@@ -378,11 +376,11 @@
 
 CRISPY_TEMPLATE_PACK = "bootstrap3"
 
-# Storing archives locally or in S3 (Package Storage settings)
+# Storing archives locally (Package Storage settings)
 
 ENABLE_DOWNLOAD_ARCHIVING = env.bool("ENABLE_DOWNLOAD_ARCHIVING", default=False)
 
-# localstorage, s3, sftp
+# localstorage configuration
 DOWNLOAD_ARCHIVING_PROVIDER = env.str(
     "DOWNLOAD_ARCHIVING_PROVIDER", default="localstorage"
 )
@@ -392,7 +390,7 @@
     "DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION", default=None
 )
 
-# Initialize the DownloadStore based on provider
+# Initialize the DownloadStore for local storage
 
 download_store = None
 if ENABLE_DOWNLOAD_ARCHIVING:
@@ -403,41 +401,6 @@
             download_store = LocalFilesystemProvider(root_path=root_path)
         except Exception as e:
             logger.error(f"Failed to initialize LocalFilesystemProvider: {e}")
-    elif DOWNLOAD_ARCHIVING_PROVIDER == "s3":
-        config = DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION or {}
-        required_keys = ["bucket_name", "aws_userid", "aws_apikey"]
-        if not all(key in config for key in required_keys):
-            logger.error(
-                f"S3 provider requires {required_keys}"
-                "in DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION"
-            )
-        else:
-            try:
-                download_store = S3LikeProvider(
-                    bucket_name=config.get("bucket_name"),
-                    aws_userid=config.get("aws_userid"),
-                    aws_apikey=config.get("aws_apikey"),
-                    other_aws_credentials=config.get("other_aws_credentials", {}),
-                )
-            except Exception as e:
-                logger.error(f"Failed to initialize S3LikeProvider: {e}")
-    elif DOWNLOAD_ARCHIVING_PROVIDER == "sftp":
-        config = DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION or {}
-        required_keys = ["host", "root_path", "ssh_credentials"]
-        if not all(key in config for key in required_keys):
-            logger.error(
-                f"SFTP provider requires {required_keys}"
-                "in DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION"
-            )
-        else:
-            try:
-                download_store = SftpProvider(
-                    host=config.get("host"),
-                    root_path=config.get("root_path"),
-                    ssh_credentials=config.get("ssh_credentials", {}),
-                )
-            except Exception as e:
-                logger.error(f"Failed to initialize SftpProvider: {e}")
     else:
         logger.error(
             f"Unknown DOWNLOAD_ARCHIVING_PROVIDER: {DOWNLOAD_ARCHIVING_PROVIDER}"

scanpipe/archiving.py

@@ -31,10 +31,6 @@
 from dataclasses import dataclass
 from pathlib import Path
 
-import boto3
-import paramiko
-from botocore.exceptions import ClientError
-from paramiko.ssh_exception import SSHException
 
 logger = logging.getLogger(__name__)
 
@@ -192,273 +188,3 @@ def find(
         return None
 
 
-class S3LikeProvider(DownloadStore):
-    def __init__(
-        self,
-        bucket_name: str,
-        aws_userid: str,
-        aws_apikey: str,
-        other_aws_credentials: dict,
-    ):
-        self.bucket_name = bucket_name
-        self.s3_client = boto3.client(
-            "s3",
-            aws_access_key_id=aws_userid,
-            aws_secret_access_key=aws_apikey,
-            **(other_aws_credentials or {}),
-        )
-
-    def _get_content_path(self, sha256: str) -> str:
-        """S3 key like 59/4c/67/<sha256>/"""
-        return f"{sha256[:2]}/{sha256[2:4]}/{sha256[4:]}/"
-
-    def list(self):
-        """List all stored downloads."""
-        downloads = []
-        try:
-            paginator = self.s3_client.get_paginator("list_objects_v2")
-            for page in paginator.paginate(Bucket=self.bucket_name):
-                for obj in page.get("Contents", []):
-                    key = obj["Key"]
-                    if key.endswith(".json"):
-                        try:
-                            response = self.s3_client.get_object(
-                                Bucket=self.bucket_name, Key=key
-                            )
-                            data = json.loads(response["Body"].read())
-                            downloads.append(Download(**data))
-                        except Exception as e:
-                            logger.error(f"Error reading S3 object {key}: {e}")
-        except ClientError as e:
-            logger.error(f"Failed to list S3 objects: {e}")
-        return downloads
-
-    def get(self, sha256_checksum: str):
-        """Retrieve a Download object for the given SHA256 hash."""
-        prefix = self._get_content_path(sha256_checksum)
-        try:
-            response = self.s3_client.list_objects_v2(
-                Bucket=self.bucket_name, Prefix=prefix, MaxKeys=1
-            )
-            if "Contents" in response:
-                key = response["Contents"][0]["Key"]
-                obj_response = self.s3_client.get_object(
-                    Bucket=self.bucket_name, Key=key
-                )
-                data = json.loads(obj_response["Body"].read())
-                return Download(**data)
-        except ClientError as e:
-            logger.error(f"Failed to get S3 object for {sha256_checksum}: {e}")
-        return None
-
-    def put(self, content: bytes, download_url: str, download_date: str, filename: str):
-        """Store the content and its metadata."""
-        sha256 = self._compute_sha256(content)
-        content_key = self._get_content_path(sha256) + "content"
-        try:
-            self.s3_client.head_object(Bucket=self.bucket_name, Key=content_key)
-            logger.info(f"Content already exists for {sha256}")
-        except ClientError:
-            try:
-                self.s3_client.put_object(
-                    Bucket=self.bucket_name,
-                    Key=content_key,
-                    Body=content,
-                )
-            except ClientError as e:
-                raise Exception(f"Failed to write content to S3 {content_key}: {e}")
-
-        origin_hash = self._compute_origin_hash(filename, download_date, download_url)
-        origin_filename = f"origin-{origin_hash}.json"
-        origin_key = self._get_content_path(sha256) + origin_filename
-
-        metadata = self._build_metadata(sha256, filename, download_date, download_url)
-        metadata_json = json.dumps(metadata, indent=2).encode("utf-8")
-        try:
-            self.s3_client.put_object(
-                Bucket=self.bucket_name,
-                Key=origin_key,
-                Body=metadata_json,
-            )
-        except ClientError as e:
-            raise Exception(f"Failed to write metadata to S3 {origin_key}: {e}")
-
-        return Download(**metadata)
-
-    def find(
-        self, download_url: str = None, filename: str = None, download_date: str = None
-    ):
-        """Find a download based on metadata."""
-        if not (download_url or filename or download_date):
-            return None
-        try:
-            paginator = self.s3_client.get_paginator("list_objects_v2")
-            for page in paginator.paginate(Bucket=self.bucket_name):
-                for obj in page.get("Contents", []):
-                    key = obj["Key"]
-                    if key.endswith(".json"):
-                        try:
-                            response = self.s3_client.get_object(
-                                Bucket=self.bucket_name, Key=key
-                            )
-                            data = json.loads(response["Body"].read())
-                            if (
-                                (
-                                    download_url is None
-                                    or data.get("url") == download_url
-                                )
-                                and (
-                                    filename is None or data.get("filename") == filename
-                                )
-                                and (
-                                    download_date is None
-                                    or data.get("download_date") == download_date
-                                )
-                            ):
-                                return Download(**data)
-                        except Exception as e:
-                            logger.error(f"Error reading S3 object {key}: {e}")
-        except ClientError as e:
-            logger.error(f"Failed to find in S3: {e}")
-        return None
-
-
-class SftpProvider(DownloadStore):
-    def __init__(self, host: str, root_path: str, ssh_credentials: dict):
-        self.host = host
-        self.root_path = Path(root_path)
-        self.ssh_credentials = ssh_credentials
-        self.ssh = paramiko.SSHClient()
-        self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        try:
-            self.ssh.connect(
-                hostname=host,
-                username=ssh_credentials.get("username"),
-                password=ssh_credentials.get("password"),
-            )
-            self.sftp = self.ssh.open_sftp()
-        except SSHException as e:
-            raise Exception(f"Failed to connect to SFTP server {host}: {e}")
-
-    def _get_content_path(self, sha256: str) -> str:
-        """SFTP path like 59/4c/67/<sha256>/"""
-        return str(self.root_path / sha256[:2] / sha256[2:4] / sha256[4:])
-
-    def list(self):
-        """List all stored downloads."""
-        downloads = []
-        try:
-            for root, _, files in self._sftp_walk(self.root_path):
-                for filename in files:
-                    if filename.endswith(".json"):
-                        file_path = os.path.join(root, filename)
-                        try:
-                            with self.sftp.open(file_path, "r") as f:
-                                data = json.load(f)
-                            downloads.append(Download(**data))
-                        except Exception as e:
-                            logger.error(f"Error reading SFTP file {file_path}: {e}")
-        except SSHException as e:
-            logger.error(f"Failed to list SFTP files: {e}")
-        return downloads
-
-    def _sftp_walk(self, path):
-        """Recursively walk SFTP directory."""
-        path = str(path)
-        for entry in self.sftp.listdir_attr(path):
-            full_path = os.path.join(path, entry.filename)
-            if stat.S_ISDIR(entry.st_mode):
-                yield from self._sftp_walk(full_path)
-            else:
-                yield path, [], [entry.filename]
-
-    def get(self, sha256_checksum: str):
-        """Retrieve a Download object for the given SHA256 hash."""
-        content_path = self._get_content_path(sha256_checksum)
-        try:
-            files = self.sftp.listdir(content_path)
-            origin_files = [
-                f for f in files if f.startswith("origin-") and f.endswith(".json")
-            ]
-            if origin_files:
-                with self.sftp.open(
-                    os.path.join(content_path, origin_files[0]), "r"
-                ) as f:
-                    data = json.load(f)
-                return Download(**data)
-        except SSHException as e:
-            logger.error(f"Failed to get SFTP file for {sha256_checksum}: {e}")
-        return None
-
-    def put(self, content: bytes, download_url: str, download_date: str, filename: str):
-        """Store the content and its metadata."""
-        sha256 = self._compute_sha256(content)
-        content_path = self._get_content_path(sha256)
-        try:
-            self.sftp.mkdir(content_path)
-        except SSHException:
-            pass
-
-        content_file = os.path.join(content_path, "content")
-        try:
-            self.sftp.stat(content_file)
-            logger.info(f"Content already exists for {sha256}")
-        except SSHException:
-            try:
-                with self.sftp.open(content_file, "wb") as f:
-                    f.write(content)
-            except SSHException as e:
-                raise Exception(f"Failed to write content to SFTP {content_file}: {e}")
-
-        origin_hash = self._compute_origin_hash(filename, download_date, download_url)
-        origin_filename = f"origin-{origin_hash}.json"
-        origin_path = os.path.join(content_path, origin_filename)
-        try:
-            self.sftp.stat(origin_path)
-            raise Exception(f"Origin {origin_filename} already exists")
-        except SSHException:
-            metadata = self._build_metadata(
-                sha256, filename, download_date, download_url
-            )
-            metadata_json = json.dumps(metadata, indent=2).encode("utf-8")
-            try:
-                with self.sftp.open(origin_path, "wb") as f:
-                    f.write(metadata_json)
-            except SSHException as e:
-                raise Exception(f"Failed to write metadata to SFTP {origin_path}: {e}")
-
-        return Download(**metadata)
-
-    def find(
-        self, download_url: str = None, filename: str = None, download_date: str = None
-    ):
-        """Find a download based on metadata."""
-        if not (download_url or filename or download_date):
-            return None
-        try:
-            for root, _, files in self._sftp_walk(self.root_path):
-                for filename in files:
-                    if filename.endswith(".json"):
-                        file_path = os.path.join(root, filename)
-                        try:
-                            with self.sftp.open(file_path, "r") as f:
-                                data = json.load(f)
-                            if (
-                                (
-                                    download_url is None
-                                    or data.get("url") == download_url
-                                )
-                                and (
-                                    filename is None or data.get("filename") == filename
-                                )
-                                and (
-                                    download_date is None
-                                    or data.get("download_date") == download_date
-                                )
-                            ):
-                                return Download(**data)
-                        except Exception as e:
-                            logger.error(f"Error reading SFTP file {file_path}: {e}")
-        except SSHException as e:
-            logger.error(f"Failed to find in SFTP: {e}")
-        return None

scanpipe/pipelines/__init__.py

@@ -34,8 +34,8 @@
 from pyinstrument import Profiler
 
 from aboutcode.pipeline import BasePipeline
-from scanpipe.settings import ENABLE_DOWNLOAD_ARCHIVING
-from scanpipe.settings import download_store
+from scancodeio.settings import ENABLE_DOWNLOAD_ARCHIVING
+from scancodeio.settings import download_store
 
 logger = logging.getLogger(__name__)
 

scanpipe/pipes/input.py

@@ -44,8 +44,8 @@
 from scanpipe.models import InputSource
 from scanpipe.pipes import scancode
 from scanpipe.pipes.output import mappings_key_by_fieldname
-from scanpipe.settings import ENABLE_DOWNLOAD_ARCHIVING
-from scanpipe.settings import download_store
+from scancodeio.settings import ENABLE_DOWNLOAD_ARCHIVING
+from scancodeio.settings import download_store
 
 logger = logging.getLogger(__name__)
 

scanpipe/tests/test_input.py

@@ -30,7 +30,7 @@
 from scanpipe.models import InputSource
 from scanpipe.pipes.input import add_input_from_upload
 from scanpipe.pipes.input import add_input_from_url
-from scanpipe.settings import download_store
+from scancodeio.settings import download_store
 from scanpipe.tests import make_project