Add support for multiple inputs in the run management command

Signed-off-by: tdruez <[email protected]>

Author: tdruez

scanpipe/management/commands/run.py

@@ -20,6 +20,7 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
+from collections import defaultdict
 from pathlib import Path
 
 from django.core.management import call_command
@@ -42,12 +43,16 @@ def add_arguments(self, parser):
             help=(
                 "One or more pipeline to run. "
                 "The pipelines executed based on their given order. "
-                'Groups can be provided using the "pipeline_name:option1,option2"'
-                " syntax."
+                'Groups can be provided using the "pipeline_name:option1,option2" '
+                "syntax."
             ),
         )
         parser.add_argument(
-            "input_location", help="Input location: file, directory, and URL supported."
+            "input_location",
+            help=(
+                "Input location: file, directory, and URL supported."
+                'Multiple values can be provided using the "input1,input2" syntax.'
+            ),
         )
         parser.add_argument("--project", required=False, help="Project name.")
         parser.add_argument(
@@ -68,22 +73,38 @@ def handle(self, *args, **options):
             "pipeline": pipelines,
             "execute": True,
             "verbosity": 0,
+            **self.get_input_options(input_location),
         }
 
-        if input_location.startswith(tuple(SCHEME_TO_FETCHER_MAPPING.keys())):
-            create_project_options["input_urls"] = [input_location]
-        else:
-            input_path = Path(input_location)
-            if not input_path.exists():
-                raise CommandError(f"{input_location} not found.")
-            if input_path.is_file():
-                create_project_options["input_files"] = [input_location]
-            else:
-                create_project_options["copy_codebase"] = input_location
-
         # Run the database migrations in case the database is not created or outdated.
         call_command("migrate", verbosity=0, interactive=False)
         # Create a project with proper inputs and execute the pipeline(s)
         call_command("create-project", project_name, **create_project_options)
         # Print the results for the specified format on stdout
         call_command("output", project=project_name, format=[output_format], print=True)
+
+    @staticmethod
+    def get_input_options(input_location):
+        """
+        Parse a comma-separated list of input locations and convert them into options
+        for the `create-project` command.
+        """
+        input_options = defaultdict(list)
+
+        for location in input_location.split(","):
+            if location.startswith(tuple(SCHEME_TO_FETCHER_MAPPING.keys())):
+                input_options["input_urls"].append(location)
+            else:
+                input_path = Path(location)
+                if not input_path.exists():
+                    raise CommandError(f"{location} not found.")
+                if input_path.is_file():
+                    input_options["input_files"].append(location)
+                else:
+                    if input_options["copy_codebase"]:
+                        raise CommandError(
+                            "Only one codebase directory can be provided as input."
+                        )
+                    input_options["copy_codebase"] = location
+
+        return input_options

scanpipe/tests/test_commands.py

@@ -984,6 +984,53 @@ def test_scanpipe_management_command_run(self):
         self.assertEqual("do_nothing", runs[1]["pipeline_name"])
         self.assertEqual(["Group1", "Group2"], runs[1]["selected_groups"])
 
+    @mock.patch("requests.sessions.Session.get")
+    def test_scanpipe_management_command_run_multiple_inputs(self, mock_get):
+        source_download_url = "https://example.com/z-source.zip#from"
+        bin_download_url = "https://example.com/z-bin.zip#to"
+        mock_get.side_effect = [
+            make_mock_response(url=source_download_url),
+            make_mock_response(url=bin_download_url),
+        ]
+
+        out = StringIO()
+        inputs = [
+            # copy_codebase option
+            str(self.data / "codebase"),
+            # input_files option
+            str(self.data / "d2d" / "jars" / "from-flume-ng-node-1.9.0.zip"),
+            str(self.data / "d2d" / "jars" / "to-flume-ng-node-1.9.0.zip"),
+            # input_urls option
+            source_download_url,
+            bin_download_url,
+        ]
+        joined_locations = ",".join(inputs)
+        with redirect_stdout(out):
+            call_command("run", "download_inputs", joined_locations)
+
+        json_data = json.loads(out.getvalue())
+        headers = json_data["headers"]
+        project_uuid = headers[0]["uuid"]
+        project = Project.objects.get(uuid=project_uuid)
+
+        expected = [
+            "from-flume-ng-node-1.9.0.zip",
+            "to-flume-ng-node-1.9.0.zip",
+            "z-bin.zip",
+            "z-source.zip",
+        ]
+        self.assertEqual(expected, sorted(project.input_files))
+
+        input_sources = headers[0]["input_sources"]
+        self.assertEqual("z-bin.zip", input_sources[2]["filename"])
+        self.assertEqual("to", input_sources[2]["tag"])
+        self.assertEqual("z-source.zip", input_sources[3]["filename"])
+        self.assertEqual("from", input_sources[3]["tag"])
+
+        codebase_files = [path.name for path in project.codebase_path.glob("*")]
+        expected = ["a.txt", "b.txt", "c.txt"]
+        self.assertEqual(expected, sorted(codebase_files))
+
     @mock.patch("scanpipe.models.Project.get_latest_output")
     @mock.patch("requests.post")
     @mock.patch("requests.sessions.Session.get")