Refactor swift

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

Author: TG1999

minecode_pipelines/pipelines/mine_swift.py

@@ -20,38 +20,96 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-from scanpipe.pipelines import Pipeline
+from datetime import datetime
 from scanpipe.pipes import federatedcode
+
 from minecode_pipelines import pipes
-from minecode_pipelines.pipes.swift import mine_and_publish_swift_packageurls
+from minecode_pipelines.pipelines import MineCodeBasePipeline
+from minecode_pipelines.pipes.swift import PACKAGE_BATCH_SIZE, mine_swift_packageurls
+from minecode_pipelines.pipes.swift import load_swift_package_urls
+from minecode_pipelines.pipelines import _mine_and_publish_packageurls
 
 
-class MineSwift(Pipeline):
+class MineComposer(MineCodeBasePipeline):
     """
-    Mine all packageURLs from a swift index and publish them to a FederatedCode repo.
+    Pipeline to mine Composer PHP packages and publish them to FederatedCode.
     """
 
+    pipeline_config_repo = "https://github.com/aboutcode-data/minecode-pipelines-config/"
+    checkpoint_path = "swift/checkpoints.json"
+    checkpoint_freq = 200
+    swift_index_repo_url = "https://github.com/SwiftPackageIndex/PackageList"
+
     @classmethod
     def steps(cls):
         return (
             cls.check_federatedcode_eligibility,
-            cls.mine_and_publish_swift_packageurls,
-            cls.delete_cloned_repos,
+            cls.create_federatedcode_working_dir,
+            cls.fetch_checkpoint_and_start_index,
+            cls.fetch_federation_config,
+            cls.clone_swift_index,
+            cls.mine_and_publish_packageurls,
+            cls.delete_working_dir,
+        )
+
+    def clone_swift_index(self):
+        """Clone the Cargo index Repo."""
+        self.swift_index_repo = federatedcode.clone_repository(
+            repo_url=self.swift_index_repo_url,
+            clone_path=self.working_path / "swift-index",
+            logger=self.log,
+        )
+
+    def fetch_checkpoint_and_start_index(self):
+        self.checkpoint_config_repo = federatedcode.clone_repository(
+            repo_url=self.pipeline_config_repo,
+            clone_path=self.working_path / "minecode-pipelines-config",
+            logger=self.log,
+        )
+        checkpoint = pipes.get_checkpoint_from_file(
+            cloned_repo=self.checkpoint_config_repo,
+            path=self.checkpoint_path,
+        )
+
+        self.start_index = checkpoint.get("start_index", 0)
+        self.log(f"start_index: {self.start_index}")
+
+    def packages_count(self):
+        return len(self.swift_packages_urls) if self.swift_packages_urls else None
+
+    def mine_packageurls(self):
+        self.swift_packages_urls = load_swift_package_urls(swift_index_repo=self.swift_index_repo)
+        return mine_swift_packageurls(
+            packages_urls=self.swift_packages_urls,
+            start_index=self.start_index,
+            logger=self.log,
         )
 
-    def check_federatedcode_eligibility(self):
-        """
-        Check if the project fulfills the following criteria for
-        pushing the project result to FederatedCode.
-        """
-        federatedcode.check_federatedcode_configured_and_available(logger=self.log)
+    def mine_and_publish_packageurls(self):
+        """Mine and publish PackageURLs."""
+        _mine_and_publish_packageurls(
+            packageurls=self.mine_packageurls(),
+            total_package_count=self.packages_count(),
+            data_cluster=self.data_cluster,
+            checked_out_repos=self.checked_out_repos,
+            working_path=self.working_path,
+            append_purls=self.append_purls,
+            commit_msg_func=self.commit_message,
+            logger=self.log,
+            checkpoint_func=self.save_check_point,
+            checkpoint_freq=self.checkpoint_freq,
+        )
 
-    def mine_and_publish_swift_packageurls(self):
-        """Mine swift package names from swift indexes or checkpoint."""
-        self.repos = mine_and_publish_swift_packageurls(self.log)
+    def save_check_point(self):
+        checkpoint = {
+            "date": str(datetime.now()),
+            "start_index": self.start_index + self.checkpoint_freq * PACKAGE_BATCH_SIZE,
+        }
 
-    def delete_cloned_repos(self):
-        pipes.delete_cloned_repos(
-            repos=self.repos,
+        self.log(f"Saving checkpoint: {checkpoint}")
+        pipes.update_checkpoints_in_github(
+            checkpoint=checkpoint,
+            cloned_repo=self.checkpoint_config_repo,
+            path=self.checkpoint_path,
             logger=self.log,
         )

minecode_pipelines/pipes/swift.py

@@ -21,37 +21,49 @@
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 import json
-import os
-from datetime import datetime
 from pathlib import Path
-from aboutcode import hashid
 from packageurl import PackageURL
 
 from minecode_pipelines.miners.swift import fetch_git_tags_raw
 from minecode_pipelines.miners.swift import get_tags_and_commits_from_git_output
 from minecode_pipelines.miners.swift import split_org_repo
+from minecode_pipelines.utils import cycle_from_index, grouper
 
-from minecode_pipelines.pipes import update_checkpoints_in_github
-from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO
-from minecode_pipelines.pipes import write_data_to_yaml_file
+PACKAGE_BATCH_SIZE = 100
 
-from minecode_pipelines.pipes import get_checkpoint_from_file
-from scanpipe.pipes.federatedcode import clone_repository
 
-from scanpipe.pipes.federatedcode import commit_and_push_changes
-from minecode_pipelines.utils import cycle_from_index
+def mine_swift_packageurls(packages_urls, start_index, logger):
+    """Mine Swift PackageURLs from package index."""
+
+    packages_iter = cycle_from_index(packages_urls, start_index)
+    for batch_index, package_batch in enumerate(
+        grouper(n=PACKAGE_BATCH_SIZE, iterable=packages_iter)
+    ):
+        for item in package_batch:
+            if not item:
+                continue
+        package_repo_url = item
+        git_ls_remote = fetch_git_tags_raw(package_repo_url, 60, logger)
+        if not git_ls_remote:
+            continue
+
+        tags_and_commits = get_tags_and_commits_from_git_output(git_ls_remote)
+        if not tags_and_commits:
+            continue
+
+        yield generate_package_urls(
+            package_repo_url=package_repo_url, tags_and_commits=tags_and_commits
+        )
 
-PACKAGE_BATCH_SIZE = 1000
-SWIFT_CHECKPOINT_PATH = "swift/checkpoints.json"
 
-MINECODE_DATA_SWIFT_REPO = os.environ.get(
-    "MINECODE_DATA_SWIFT_REPO", "https://github.com/aboutcode-data/minecode-data-swift-test"
-)
-MINECODE_SWIFT_INDEX_REPO = "https://github.com/SwiftPackageIndex/PackageList"
+def load_swift_package_urls(swift_index_repo):
+    packages_path = Path(swift_index_repo.working_dir) / "packages.json"
+    with open(packages_path) as f:
+        packages_urls = json.load(f)
+    return packages_urls
 
 
-def store_swift_packages(package_repo_url, tags_and_commits, cloned_data_repo):
-    """Collect Swift package versions into purls and write them to the repo."""
+def generate_package_urls(package_repo_url, tags_and_commits):
     org, name = split_org_repo(package_repo_url)
     org = "github.com/" + org
     base_purl = PackageURL(type="swift", namespace=org, name=name)
@@ -70,102 +82,4 @@ def store_swift_packages(package_repo_url, tags_and_commits, cloned_data_repo):
         if purl:
             updated_purls.append(purl)
 
-    purl_yaml_path = cloned_data_repo.working_dir / hashid.get_package_purls_yml_file_path(
-        base_purl
-    )
-    write_data_to_yaml_file(path=purl_yaml_path, data=updated_purls)
-    return purl_yaml_path, base_purl
-
-
-def mine_and_publish_swift_packageurls(logger):
-    """
-    Clone Swift-related repositories, process Swift packages, and publish their
-    Package URLs (purls) to the data repository.
-
-    This function:
-      1. Clones the Swift index, data, and pipelines config repositories.
-      2. Loads the list of Swift package repositories from `packages.json`.
-      3. Iterates over each package, fetching tags/commits and generating purls.
-      4. Commits and pushes purl files to the data repository in batches.
-      5. Updates checkpoint information in the config repository to track progress.
-
-    logger (callable): Optional logging function for status updates.
-    Returns: list: A list of cloned repository objects in the order:
-    [swift_index_repo, cloned_data_repo, cloned_config_repo]
-    """
-
-    swift_index_repo = clone_repository(MINECODE_SWIFT_INDEX_REPO)
-    cloned_data_repo = clone_repository(MINECODE_DATA_SWIFT_REPO)
-    cloned_config_repo = clone_repository(MINECODE_PIPELINES_CONFIG_REPO)
-
-    if logger:
-        logger(f"{MINECODE_SWIFT_INDEX_REPO} repo cloned at: {swift_index_repo.working_dir}")
-        logger(f"{MINECODE_DATA_SWIFT_REPO} repo cloned at: {cloned_data_repo.working_dir}")
-        logger(f"{MINECODE_PIPELINES_CONFIG_REPO} repo cloned at: {cloned_config_repo.working_dir}")
-
-    packages_path = Path(swift_index_repo.working_dir) / "packages.json"
-    with open(packages_path) as f:
-        packages_urls = json.load(f)
-
-    counter = 0
-    purl_files = []
-    purls = []
-
-    swift_checkpoint = get_checkpoint_from_file(
-        cloned_repo=cloned_config_repo, path=SWIFT_CHECKPOINT_PATH
-    )
-
-    start_index = swift_checkpoint.get("start_index", 0)
-
-    if logger:
-        logger(f"Processing total files: {len(packages_urls)}")
-
-    for idx, package_repo_url in enumerate(cycle_from_index(packages_urls, start_index)):
-        git_ls_remote = fetch_git_tags_raw(package_repo_url, 60, logger)
-        if not git_ls_remote:
-            continue
-
-        tags_and_commits = get_tags_and_commits_from_git_output(git_ls_remote)
-        if not tags_and_commits:
-            continue
-
-        purl_file, base_purl = store_swift_packages(
-            package_repo_url, tags_and_commits, cloned_data_repo
-        )
-
-        purl_files.append(purl_file)
-        purls.append(str(base_purl))
-        counter += 1
-
-        if counter >= PACKAGE_BATCH_SIZE:
-            if purls and purl_files:
-                logger(f"Committing packageURLs: {', '.join(purls)}")
-                commit_and_push_changes(
-                    repo=cloned_data_repo, files_to_commit=purl_files, purls=purls, logger=logger
-                )
-
-            purl_files = []
-            purls = []
-            counter = 0
-
-            if start_index == idx:
-                continue
-
-            settings_data = {
-                "date": str(datetime.now()),
-                "start_index": idx,
-            }
-
-            update_checkpoints_in_github(
-                checkpoint=settings_data,
-                cloned_repo=cloned_config_repo,
-                path=SWIFT_CHECKPOINT_PATH,
-            )
-
-    if purls and purl_files:
-        logger(f"Committing packageURLs: {', '.join(purls)}")
-        commit_and_push_changes(
-            repo=cloned_data_repo, files_to_commit=purl_files, purls=purls, logger=logger
-        )
-
-    return [swift_index_repo, cloned_data_repo, cloned_config_repo]
+    return base_purl, updated_purls

minecode_pipelines/tests/pipes/test_swift.py

@@ -13,7 +13,6 @@
 from unittest.mock import Mock, patch
 import saneyaml
 from minecode_pipelines.pipes.swift import (
-    store_swift_packages,
     get_tags_and_commits_from_git_output,
 )
 
@@ -22,7 +21,7 @@
 
 class SwiftPipelineTests(TestCase):
     def _run_package_test(
-        self, mock_write, package_repo_url, commits_tags_file, expected_file, expected_path_parts
+        self, package_repo_url, commits_tags_file, expected_file, expected_path_parts
     ):
         # Load test input and expected output
         with open(commits_tags_file, encoding="utf-8") as f:
@@ -37,20 +36,7 @@ def _run_package_test(
 
             # Execute function under test
             tags_and_commits = get_tags_and_commits_from_git_output(git_ls_remote)
-            store_swift_packages(package_repo_url, tags_and_commits, repo)
 
-            # Verify function call
-            mock_write.assert_called_once()
-            _, kwargs = mock_write.call_args
-            base_purl, written_packages = kwargs["path"], kwargs["data"]
-
-            # Expected file path
-            expected_base_purl = Path(tmpdir).joinpath(*expected_path_parts)
-
-            self.assertEqual(str(base_purl), str(expected_base_purl))
-            self.assertEqual(written_packages, expected)
-
-    @patch("minecode_pipelines.pipes.swift.write_data_to_yaml_file")
     def test_swift_safe_collection_access(self, mock_write):
         self._run_package_test(
             mock_write,
@@ -67,10 +53,8 @@ def test_swift_safe_collection_access(self, mock_write):
             ],
         )
 
-    @patch("minecode_pipelines.pipes.swift.write_data_to_yaml_file")
-    def test_human_string(self, mock_write):
+    def test_human_string(self):
         self._run_package_test(
-            mock_write,
             package_repo_url="https://github.com/zonble/HumanString.git",
             commits_tags_file=DATA_DIR / "commits_tags2.txt",
             expected_file=DATA_DIR / "expected2.yaml",
@@ -84,10 +68,8 @@ def test_human_string(self, mock_write):
             ],
         )
 
-    @patch("minecode_pipelines.pipes.swift.write_data_to_yaml_file")
-    def test_swift_financial(self, mock_write):
+    def test_swift_financial(self):
         self._run_package_test(
-            mock_write,
             package_repo_url="https://github.com/zrluety/SwiftFinancial.git",
             commits_tags_file=DATA_DIR / "commits_tags3.txt",
             expected_file=DATA_DIR / "expected3.yaml",
@@ -101,10 +83,8 @@ def test_swift_financial(self, mock_write):
             ],
         )
 
-    @patch("minecode_pipelines.pipes.swift.write_data_to_yaml_file")
-    def test_swift_xcf_sodium(self, mock_write):
+    def test_swift_xcf_sodium(self):
         self._run_package_test(
-            mock_write,
             package_repo_url="https://github.com/0xacdc/XCFSodium",
             commits_tags_file=DATA_DIR / "commits_tags4.txt",
             expected_file=DATA_DIR / "expected4.yaml",