fix CI errors and imports

Signed-off-by: Varsha U N <[email protected]>

Author: VarshaUN

scanpipe/pipelines/__init__.py

@@ -25,7 +25,6 @@
 import logging
 import traceback
 from contextlib import contextmanager
-from datetime import datetime
 from functools import wraps
 from pathlib import Path
 
@@ -36,7 +35,6 @@
 from pyinstrument import Profiler
 
 from aboutcode.pipeline import BasePipeline
-from scancodeio.settings import download_store
 
 logger = logging.getLogger(__name__)
 
@@ -180,6 +178,7 @@ def download_missing_inputs(self):
         if error_tracebacks:
             raise InputFilesError(error_tracebacks)
 
+
 class ProjectPipeline(CommonStepsMixin, BasePipeline):
     """Main class for all project related pipelines including common steps methods."""
 

scanpipe/pipes/input.py

@@ -23,7 +23,6 @@
 import logging
 import os
 import shutil
-from datetime import datetime
 from pathlib import Path
 
 from django.core.exceptions import FieldDoesNotExist
@@ -34,7 +33,6 @@
 import requests
 from typecode.contenttype import get_type
 
-from scancodeio.settings import download_store
 from scanpipe import pipes
 from scanpipe.models import CodebaseRelation
 from scanpipe.models import CodebaseResource
@@ -261,43 +259,21 @@ def add_input_from_url(project, url, filename=None):
         raise
 
     filename = filename or url.split("/")[-1] or "downloaded_file"
+    input_path = project.input_path / filename
 
-    if download_store:
-        try:
-            download = download_store.put(
-                content=content,
-                download_url=url,
-                download_date=datetime.now().isoformat(),
-                filename=filename,
-            )
-            InputSource.objects.create(
-                project=project,
-                sha256=download.sha256,
-                download_url=download.download_url,
-                filename=download.filename,
-                download_date=download.download_date,
-                file_path=str(download.path),
-                is_uploaded=False,
-            )
-        except Exception as e:
-            logger.error(f"Failed to archive download for {url}: {e}")
-            raise
-    else:
-        input_path = project.input_path / filename
-        try:
-            input_path.parent.mkdir(parents=True, exist_ok=True)
-            with open(input_path, "wb") as f:
-                f.write(content)
-            InputSource.objects.create(
-                project=project,
-                filename=filename,
-                download_url=url,
-                file_path=str(input_path),
-                is_uploaded=False,
-            )
-        except Exception as e:
-            logger.error(f"Failed to save {filename} to {input_path}: {e}")
-            raise
+    try:
+        input_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(input_path, "wb") as f:
+            f.write(content)
+        InputSource.objects.create(
+            project=project,
+            filename=filename,
+            download_url=url,
+            is_uploaded=False,
+        )
+    except Exception as e:
+        logger.error(f"Failed to save {filename} to {input_path}: {e}")
+        raise
 
 
 def add_input_from_upload(project, uploaded_file):
@@ -307,39 +283,16 @@ def add_input_from_upload(project, uploaded_file):
     """
     content = uploaded_file.read()
     filename = uploaded_file.name
-
-    if download_store:
-        try:
-            download = download_store.put(
-                content=content,
-                download_url="",
-                download_date=datetime.now().isoformat(),
-                filename=filename,
-            )
-            InputSource.objects.create(
-                project=project,
-                sha256=download.sha256,
-                download_url=download.download_url,
-                filename=download.filename,
-                download_date=download.download_date,
-                file_path=str(download.path),
-                is_uploaded=True,
-            )
-        except Exception as e:
-            logger.error(f"Failed to archive upload {filename}: {e}")
-            raise
-    else:
-        input_path = project.input_path / filename
-        try:
-            input_path.parent.mkdir(parents=True, exist_ok=True)
-            with open(input_path, "wb") as f:
-                f.write(content)
-            InputSource.objects.create(
-                project=project,
-                filename=filename,
-                file_path=str(input_path),
-                is_uploaded=True,
-            )
-        except Exception as e:
-            logger.error(f"Failed to save {filename} to {input_path}: {e}")
-            raise
+    input_path = project.input_path / filename
+    try:
+        input_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(input_path, "wb") as f:
+            f.write(content)
+        InputSource.objects.create(
+            project=project,
+            filename=filename,
+            is_uploaded=True,
+        )
+    except Exception as e:
+        logger.error(f"Failed to save {filename} to {input_path}: {e}")
+        raise

scanpipe/tests/test_input.py

@@ -25,11 +25,11 @@
 
 
 from pathlib import Path
+from unittest.mock import Mock
 from unittest.mock import patch
 
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.test import TestCase
-from django.conf import settings
 
 from scanpipe.models import InputSource
 from scanpipe.pipes.input import add_input_from_upload
@@ -49,64 +49,46 @@ def setUp(self):
 
     @patch("requests.get")
     def test_add_input_from_url(self, mock_get):
-        test_url = "https://files.pythonhosted.org/packages/sample.tar.gz"
-        mock_get.return_value.content = self.test_content
-        mock_get.return_value.status_code = 200
+        test_url = "https://example.com/test.tar.gz"
+        mock_response = Mock()
+        mock_response.content = self.test_content
+        mock_response.raise_for_status.return_value = None
+        mock_get.return_value = mock_response
         add_input_from_url(self.project, test_url, filename=self.test_filename)
         input_source = InputSource.objects.get(project=self.project)
-        self.assertEqual(input_source.filename, self.test_filename)
         self.assertEqual(input_source.download_url, test_url)
-        self.assertTrue(input_source.sha256)
-        self.assertTrue(input_source.download_date)
-        self.assertFalse(input_source.is_uploaded)
-        self.assertTrue(
-            input_source.file_path.startswith(settings.CENTRAL_ARCHIVE_PATH)
-        )
-        self.assertTrue(Path(input_source.file_path).exists())
-
-    @patch("scanpipe.pipes.input.download_store", None)
-    @patch("requests.get")
-    def test_add_input_from_url_fallback(self, mock_get):
-        test_url = "https://files.pythonhosted.org/packages/sample.tar.gz"
-        mock_get.return_value.content = self.test_content
-        mock_get.return_value.status_code = 200
-        add_input_from_url(self.project, test_url, filename=self.test_filename)
-        input_source = InputSource.objects.get(project=self.project)
         self.assertEqual(input_source.filename, self.test_filename)
-        self.assertEqual(input_source.download_url, test_url)
-        self.assertFalse(input_source.sha256)
-        self.assertFalse(input_source.download_date)
         self.assertFalse(input_source.is_uploaded)
-        self.assertTrue(
-            str(input_source.file_path).startswith(str(self.project.input_path))
-        )
-        self.assertTrue(Path(input_source.file_path).exists())
+        self.assertTrue((self.project.input_path / self.test_filename).exists())
 
     def test_add_input_from_upload(self):
         uploaded_file = SimpleUploadedFile(self.test_filename, self.test_content)
         add_input_from_upload(self.project, uploaded_file)
         input_source = InputSource.objects.get(project=self.project)
         self.assertEqual(input_source.filename, self.test_filename)
         self.assertEqual(input_source.download_url, "")
-        self.assertTrue(input_source.sha256)
-        self.assertTrue(input_source.download_date)
         self.assertTrue(input_source.is_uploaded)
-        self.assertTrue(
-            input_source.file_path.startswith(settings.CENTRAL_ARCHIVE_PATH)
-        )
-        self.assertTrue(Path(input_source.file_path).exists())
+        self.assertTrue((self.project.input_path / self.test_filename).exists())
+
+    @patch("requests.get")
+    def test_add_input_from_url_fallback(self, mock_get):
+        test_url = "https://example.com/test.tar.gz"
+        mock_response = Mock()
+        mock_response.content = self.test_content
+        mock_response.raise_for_status.return_value = None
+        mock_get.return_value = mock_response
+        add_input_from_url(self.project, test_url, filename=self.test_filename)
+        input_source = InputSource.objects.get(project=self.project)
+        self.assertEqual(input_source.download_url, test_url)
+        self.assertEqual(input_source.filename, self.test_filename)
+        self.assertFalse(input_source.is_uploaded)
+        self.assertTrue((self.project.input_path / self.test_filename).exists())
 
-    @patch("scanpipe.pipes.input.download_store", None)
     def test_add_input_from_upload_fallback(self):
         uploaded_file = SimpleUploadedFile(self.test_filename, self.test_content)
         add_input_from_upload(self.project, uploaded_file)
         input_source = InputSource.objects.get(project=self.project)
         self.assertEqual(input_source.filename, self.test_filename)
         self.assertEqual(input_source.download_url, "")
-        self.assertFalse(input_source.sha256)
-        self.assertFalse(input_source.download_date)
         self.assertTrue(input_source.is_uploaded)
-        self.assertTrue(
-            str(input_source.file_path).startswith(str(self.project.input_path))
-        )
-        self.assertTrue(Path(input_source.file_path).exists())
+        self.assertTrue((self.project.input_path / self.test_filename).exists())

scanpipe/tests/test_pipelines.py

@@ -41,7 +41,6 @@
 from scanpipe import pipes
 from scanpipe.models import CodebaseResource
 from scanpipe.models import DiscoveredPackage
-from scanpipe.models import InputSource
 from scanpipe.pipelines import CommonStepsMixin
 from scanpipe.pipelines import InputFilesError
 from scanpipe.pipelines import Pipeline
@@ -185,7 +184,7 @@ def test_scanpipe_pipeline_class_download_inputs_attribute(self):
         run = project1.add_pipeline("download_inputs")
         pipeline = run.make_pipeline_instance()
         self.assertTrue(pipeline.download_inputs)
-        expected = (CommonStepsMixin.download_missing_inputs)
+        expected = (CommonStepsMixin.download_missing_inputs,)
         self.assertEqual(expected, pipeline.get_initial_steps())
         expected = (CommonStepsMixin.download_missing_inputs, DownloadInput.step1)
         self.assertEqual(expected, pipeline.get_steps())
@@ -287,57 +286,6 @@ def mock_make_to_path(**kwargs):
         self.assertTrue(input_source.exists())
 
     @mock.patch("requests.get")
-    def test_archive_downloads(self, mock_get):
-        project1 = make_project()
-        run = project1.add_pipeline("scan_codebase")
-        pipeline = run.make_pipeline_instance()
-        test_filename = "sample.tar.gz"
-        test_url = "https://files.pythonhosted.org/packages/sample.tar.gz"
-        test_data_path = (
-            Path(__file__).parent / "data" / "test-downloads" / test_filename
-        )
-        with open(test_data_path, "rb") as f:
-            test_content = f.read()
-
-        input_source = InputSource.objects.create(
-            InputSource.objects.create(
-                project1=project1,
-                filename=test_filename,
-                download_url=test_url,
-                is_uploaded=False,
-            )
-        )
-        mock_get.return_value.content = test_content
-        mock_get.return_value.status_code = 200
-
-        pipeline.download_missing_inputs()
-        input_source.refresh_from_db()
-        self.assertTrue(
-            input_source.file_path.startswith(settings.CENTRAL_ARCHIVE_PATH)
-        )
-        self.assertTrue(Path(input_source.file_path).exists())
-
-        pipeline.archive_downloads()
-        input_source = InputSource.refresh_from_db()
-        self.assertTrue(input_source.sha256)
-        self.assertTrue(input_source.download_date)
-        self.assertEqual(input_source.download_url, test_url)
-        self.assertEqual(input_source.filename, test_filename)
-
-        project2 = make_project(name="project2")
-        input_source2 = InputSource.objects.create(
-            project=project2,
-            filename=test_filename,
-            download_url=test_url,
-            is_uploaded=False,
-        )
-        run2 = project2.add_pipeline("scan_codebase")
-        pipeline2 = run2.make_pipeline_instance()
-        pipeline2.download_missing_inputs()
-        input_source2.refresh_from_db()
-        self.assertEqual(input_source.file_path, input_source2.file_path)
-        self.assertTrue(Path(input_source2.file_path).exists())
-
     def test_scanpipe_pipeline_class_save_errors_context_manager(self):
         project1 = make_project()
         run = project1.add_pipeline("do_nothing")