add support for extra extensions

Signed-off-by: Varsha U N <[email protected]>

Author: VarshaUN

scanpipe/pipelines/analyze_docker.py

@@ -21,13 +21,9 @@
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 import logging
-<<<<<<< HEAD
-from scanpipe.pipelines.analyze_root_filesystem import RootFS
-=======
 from pathlib import Path
 
-from scanpipe.pipelines import RootFS
->>>>>>> ca3a1ac0c0147a6f3f59999a67bf586eab9b8a36
+from scanpipe.pipelines.analyze_root_filesystem import RootFS
 from scanpipe.pipes import docker
 from scanpipe.pipes import rootfs
 from scanpipe.pipes.fetch import store_package_archive

scanpipe/pipelines/analyze_docker_windows.py

@@ -21,17 +21,14 @@
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 import logging
-<<<<<<< HEAD
-from scanpipe.pipelines.analyze_docker import Docker
-=======
 from pathlib import Path
 
-from scanpipe.pipelines import Docker
->>>>>>> ca3a1ac0c0147a6f3f59999a67bf586eab9b8a36
+from scanpipe.pipelines.analyze_docker import Docker
 from scanpipe.pipes import docker
 from scanpipe.pipes import rootfs
 from scanpipe.pipes import windows
 from scanpipe.pipes.fetch import store_package_archive
+from scanpipe.pipes.input import is_archive
 
 logger = logging.getLogger(__name__)
 class DockerWindows(Docker):
@@ -71,8 +68,8 @@ def store_package_archives(self):
 
         package_files = [
             resource.path
-            for resource in self.project.codebaseresources.filter(
-                extension__in=[".msi", ".exe"])
+            for resource in self.project.codebaseresources.all()
+            if is_archive(resource.path)
         ]
 
         for package_path in package_files:

scanpipe/pipelines/analyze_root_filesystem.py

@@ -30,6 +30,7 @@
 from scanpipe.pipes import rootfs
 from scanpipe.pipes import scancode
 from scanpipe.pipes.fetch import store_package_archive
+from scanpipe.pipes.input import is_archive
 
 
 class RootFS(Pipeline):
@@ -110,9 +111,8 @@ def store_package_archives(self):
 
         package_files = [
             resource.path
-            for resource in self.project.codebaseresources.filter(
-                extension__in=[".deb", ".apk"]
-            )
+            for resource in self.project.codebaseresources.all()
+            if is_archive(resource.path)
         ]
         for package_path in package_files:
             if not Path(package_path).exists():

scanpipe/pipelines/deploy_to_develop.py

@@ -21,6 +21,8 @@
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 import logging
+from pathlib import Path
+
 from aboutcode.pipeline import optional_step
 from scanpipe import pipes
 from scanpipe.pipelines import Pipeline
@@ -32,6 +34,7 @@
 from scanpipe.pipes import purldb
 from scanpipe.pipes import scancode
 from scanpipe.pipes.fetch import store_package_archive
+from scanpipe.pipes.input import is_archive
 
 logger = logging.getLogger(__name__)
 
@@ -142,9 +145,8 @@ def store_package_archives(self):
         stored_files = []
         package_files = [
             resource.path
-            for resource in self.project.codebaseresources.filter(
-                extension__in=self.purldb_package_extensions
-            )
+            for resource in self.project.codebaseresources.all()
+            if is_archive(resource.path)
         ]
 
         for package_path in package_files:

scanpipe/pipelines/inspect_packages.py

@@ -21,16 +21,10 @@
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 import logging
-<<<<<<< HEAD
-from aboutcode.pipeline import optional_step
-=======
-from pathlib import Path
 
 from aboutcode.pipeline import group
->>>>>>> ca3a1ac0c0147a6f3f59999a67bf586eab9b8a36
 from scanpipe.pipelines.scan_codebase import ScanCodebase
 from scanpipe.pipes import scancode
-from scanpipe.pipes.fetch import store_package_archive
 
 logger = logging.getLogger(__name__)
 
@@ -75,40 +69,6 @@ def scan_for_application_packages(self):
             progress_logger=self.log,
         )
 
-    def store_package_archives(self):
-        """Store identified package archives locally if enabled."""
-        if not self.project.use_local_storage:
-            logger.info(f"Local storage is disabled for project: {self.project.name}."
-                         "Skipping package storage.")
-            return []
-
-        logger.info(f"Storing package archives for project: {self.project.name}")
-        stored_files = []
-        package_files = [
-            resource.path
-            for resource in self.project.codebaseresources.filter(
-                extension__in=[
-                    ".zip", ".whl", ".tar.gz", ".deb", ".rpm", ".apk", ".nupkg", ".msi",
-                      ".exe"]
-            )
-        ]
-
-        for package_path in package_files:
-            if not Path(package_path).exists():
-                logger.error(f"Invalid or missing package path: {package_path}")
-                continue
-            package_path_str = str(package_path)
-            logger.info(f"Storing package archive: {package_path_str}")
-            try:
-                result = store_package_archive(
-                    self.project, url=None, file_path=package_path_str
-                )
-                logger.info(f"Stored package archive {package_path_str}: {result}")
-                stored_files.append(result)
-            except Exception as e:
-                logger.error(f"Failed to store {package_path_str}: {e}")
-
-        return stored_files
 
     @group("StaticResolver")
     def resolve_dependencies(self):

scanpipe/pipelines/resolve_dependencies.py

@@ -21,16 +21,15 @@
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 import logging
-<<<<<<< HEAD
-=======
 from pathlib import Path
->>>>>>> ca3a1ac0c0147a6f3f59999a67bf586eab9b8a36
-from aboutcode.pipeline import optional_step
+
 from aboutcode.pipeline import group
+from aboutcode.pipeline import optional_step
 from scanpipe.pipelines.scan_codebase import ScanCodebase
 from scanpipe.pipes import resolve
 from scanpipe.pipes import scancode
 from scanpipe.pipes.fetch import store_package_archive
+from scanpipe.pipes.input import is_archive
 
 logger = logging.getLogger(__name__)
 class ResolveDependencies(ScanCodebase):
@@ -119,11 +118,8 @@ def store_package_archives(self):
         stored_files = []
         package_files = [
             resource.path
-            for resource in self.project.codebaseresources.filter(
-                extension__in=[
-                    ".whl", ".tar.gz", ".zip", ".deb", ".rpm", ".apk", ".nupkg"
-                    ]
-            )
+            for resource in self.project.codebaseresources.all()
+            if is_archive(resource.path)
         ]
 
         for package_path in package_files:

scanpipe/pipelines/scan_codebase.py

@@ -28,6 +28,7 @@
 from scanpipe.pipes import scancode
 from scanpipe.pipes.fetch import store_package_archive
 from scanpipe.pipes.input import copy_inputs
+from scanpipe.pipes.input import is_archive
 
 logger = logging.getLogger(__name__)
 
@@ -71,11 +72,8 @@ def store_package_archives(self):
         stored_files = []
         package_files = [
             resource.path
-            for resource in self.project.codebaseresources.filter(
-                extension__in=[
-                    ".whl", ".tar.gz", ".zip", ".deb", ".rpm", ".apk", ".nupkg", ".msi",
-                    ".exe"]
-            )
+            for resource in self.project.codebaseresources.all()
+            if is_archive(resource.path)
         ]
 
         for package_path in package_files:

scanpipe/pipes/fetch.py

@@ -28,6 +28,7 @@
 import re
 import subprocess
 import tempfile
+from cgi import parse_header_parameters
 from collections import namedtuple
 from pathlib import Path
 from urllib.parse import unquote
@@ -41,8 +42,6 @@
 from commoncode import command
 from commoncode.hash import multi_checksums
 from commoncode.text import python_safe_name
-from packageurl import PackageURL
-from packageurl.contrib import purl2url
 from plugincode.location_provider import get_location
 from requests import auth as request_auth
 
@@ -363,10 +362,6 @@ def fetch_git_repo(url, to=None):
     )
 
 
-<<<<<<< HEAD
-=======
-
->>>>>>> ca3a1ac0c0147a6f3f59999a67bf586eab9b8a36
 def store_package_archive(project, url=None, file_path=None):
     """
     Store a package in PackageArchive and link it to DownloadedPackage.
@@ -441,10 +436,6 @@ def store_package_archive(project, url=None, file_path=None):
         return None
 
 
-<<<<<<< HEAD
-=======
-
->>>>>>> ca3a1ac0c0147a6f3f59999a67bf586eab9b8a36
 SCHEME_TO_FETCHER_MAPPING = {
     "http": fetch_http,
     "https": fetch_http,
@@ -461,7 +452,7 @@ def get_fetcher(url):
         return fetch_git_repo
 
     if url.startswith("pkg:"):
-        return fetch_package_url
+        return fetch_url
 
     # Not using `urlparse(url).scheme` for the scheme as it converts to lower case.
     scheme = url.split("://")[0]