Remove checkpoint from swift mining

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

Author: TG1999

minecode_pipelines/pipelines/mine_swift.py

@@ -20,32 +20,25 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-from datetime import datetime
 from scanpipe.pipes import federatedcode
 
-from minecode_pipelines import pipes
 from minecode_pipelines.pipelines import MineCodeBasePipeline
-from minecode_pipelines.pipes.swift import PACKAGE_BATCH_SIZE, mine_swift_packageurls
+from minecode_pipelines.pipes.swift import mine_swift_packageurls
 from minecode_pipelines.pipes.swift import load_swift_package_urls
-from minecode_pipelines.pipelines import _mine_and_publish_packageurls
 
 
 class MineSwift(MineCodeBasePipeline):
     """
     Pipeline to mine Swift packages and publish them to FederatedCode.
     """
 
-    pipeline_config_repo = "https://github.com/aboutcode-data/minecode-pipelines-config/"
-    checkpoint_path = "swift/checkpoints.json"
-    checkpoint_freq = 30
     swift_index_repo_url = "https://github.com/SwiftPackageIndex/PackageList"
 
     @classmethod
     def steps(cls):
         return (
             cls.check_federatedcode_eligibility,
             cls.create_federatedcode_working_dir,
-            cls.fetch_checkpoint_and_start_index,
             cls.fetch_federation_config,
             cls.clone_swift_index,
             cls.mine_and_publish_packageurls,
@@ -60,20 +53,6 @@ def clone_swift_index(self):
             logger=self.log,
         )
 
-    def fetch_checkpoint_and_start_index(self):
-        self.checkpoint_config_repo = federatedcode.clone_repository(
-            repo_url=self.pipeline_config_repo,
-            clone_path=self.working_path / "minecode-pipelines-config",
-            logger=self.log,
-        )
-        checkpoint = pipes.get_checkpoint_from_file(
-            cloned_repo=self.checkpoint_config_repo,
-            path=self.checkpoint_path,
-        )
-
-        self.start_index = checkpoint.get("start_index", 0)
-        self.log(f"start_index: {self.start_index}")
-
     def packages_count(self):
         return len(self.swift_packages_urls) if self.swift_packages_urls else None
 
@@ -82,35 +61,5 @@ def mine_packageurls(self):
         self.log(f"Total Swift packages to process: {len(self.swift_packages_urls)}")
         return mine_swift_packageurls(
             packages_urls=self.swift_packages_urls,
-            start_index=self.start_index,
-            logger=self.log,
-        )
-
-    def mine_and_publish_packageurls(self):
-        """Mine and publish PackageURLs."""
-        _mine_and_publish_packageurls(
-            packageurls=self.mine_packageurls(),
-            total_package_count=self.packages_count(),
-            data_cluster=self.data_cluster,
-            checked_out_repos=self.checked_out_repos,
-            working_path=self.working_path,
-            append_purls=self.append_purls,
-            commit_msg_func=self.commit_message,
-            logger=self.log,
-            checkpoint_func=self.save_check_point,
-            checkpoint_freq=self.checkpoint_freq,
-        )
-
-    def save_check_point(self):
-        checkpoint = {
-            "date": str(datetime.now()),
-            "start_index": self.start_index + self.checkpoint_freq * PACKAGE_BATCH_SIZE,
-        }
-
-        self.log(f"Saving checkpoint: {checkpoint}")
-        pipes.update_checkpoints_in_github(
-            checkpoint=checkpoint,
-            cloned_repo=self.checkpoint_config_repo,
-            path=self.checkpoint_path,
             logger=self.log,
         )

minecode_pipelines/pipes/swift.py

@@ -24,36 +24,27 @@
 from pathlib import Path
 from packageurl import PackageURL
 
-from minecode_pipelines.utils import cycle_from_index, grouper
 import shutil
 import subprocess
 from urllib.parse import urlparse
 
-PACKAGE_BATCH_SIZE = 100
 
-
-def mine_swift_packageurls(packages_urls, start_index, logger):
+def mine_swift_packageurls(packages_urls, logger):
     """Mine Swift PackageURLs from package index."""
 
-    packages_iter = cycle_from_index(packages_urls, start_index)
-    for batch_index, package_batch in enumerate(
-        grouper(n=PACKAGE_BATCH_SIZE, iterable=packages_iter)
-    ):
-        for package_repo_url in package_batch:
-            if not package_repo_url:
-                continue
-            logger(f"Processing package repo URL: {package_repo_url}")
-            git_ls_remote = fetch_git_tags_raw(package_repo_url, 60, logger)
-            if not git_ls_remote:
-                continue
-
-            tags_and_commits = get_tags_and_commits_from_git_output(git_ls_remote)
-            if not tags_and_commits:
-                continue
-
-            yield generate_package_urls(
-                package_repo_url=package_repo_url, tags_and_commits=tags_and_commits, logger=logger
-            )
+    for package_repo_url in packages_urls:
+        logger(f"Processing package repo URL: {package_repo_url}")
+        git_ls_remote = fetch_git_tags_raw(package_repo_url, 60, logger)
+        if not git_ls_remote:
+            continue
+
+        tags_and_commits = get_tags_and_commits_from_git_output(git_ls_remote)
+        if not tags_and_commits:
+            continue
+
+        yield generate_package_urls(
+            package_repo_url=package_repo_url, tags_and_commits=tags_and_commits, logger=logger
+        )
 
 
 def load_swift_package_urls(swift_index_repo):