Implement functionality to process bulk of commits

Signed-off-by: ziad hany <[email protected]>

Author: ziadhany

minecode_pipelines/miners/cargo.py

@@ -6,47 +6,61 @@
 # See https://github.com/aboutcode-org/purldb for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-from minecode_pipelines.pipes import get_last_commit, get_changed_files, update_last_commit
+from minecode_pipelines.pipes import get_last_commit
+from minecode_pipelines.pipes import get_changed_files
+from minecode_pipelines.pipes import update_last_commit
 from minecode_pipelines.pipes.cargo import store_cargo_packages
 import json
 from pathlib import Path
 
+from minecode_pipelines.utils import get_next_x_commit
+
+
+def process_cargo_packages(cargo_repo, fed_repo, fed_conf_repo, logger):
+    """
+    Process Cargo index files commit by commit.
+    Push changes to fed_repo after:
+    - every `commit_batch_size` commits, OR
+    - every `file_batch_size` files, OR
+    - when reaching HEAD.
+    """
 
-def process_cargo_packages(cargo_repo, fed_repo, logger):
     base_path = Path(cargo_repo.working_tree_dir)
-    setting_last_commit = get_last_commit(fed_repo, "cargo")
-    valid_files = get_changed_files(cargo_repo, setting_last_commit)  # start from empty tree hash
 
-    logger(f"Found {len(valid_files)} changed files in Cargo index.")
-    targets_files = []
-    for file_path in base_path.glob("**/*"):
-        if not file_path.is_file():
-            continue
+    while True:
+        setting_last_commit = get_last_commit(fed_conf_repo, "cargo")
+
+        if setting_last_commit is None:
+            setting_last_commit = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
 
-        rel_path = str(file_path.relative_to(base_path))
-        if rel_path not in valid_files:
-            continue
+        next_commit = get_next_x_commit(cargo_repo, setting_last_commit, x=1000, branch="master")
 
-        if file_path.name in {"config.json", "README.md", "update-dl-url.yml"}:
-            continue
+        if next_commit == setting_last_commit:
+            logger("No new commits to mine")
+            break
 
-        targets_files.append(file_path)
+        changed_files = get_changed_files(
+            cargo_repo, commit_x=setting_last_commit, commit_y=next_commit
+        )
+        logger(f"Found {len(changed_files)} changed files in Cargo index.")
 
-    logger(f"Collected {len(targets_files)} target package files to process.")
+        for idx, rel_path in enumerate(changed_files):
+            file_path = base_path / rel_path
+            logger(f"Found {file_path}.")
 
-    for idx, file_path in enumerate(targets_files, start=1):
-        packages = []
-        with open(file_path, encoding="utf-8") as f:
-            for line in f:
-                if line.strip():
-                    packages.append(json.loads(line))
+            if not file_path.is_file():
+                continue
 
-        if not packages:
-            continue
+            if file_path.name in {"config.json", "README.md", "update-dl-url.yml"}:
+                continue
+            packages = []
+            with open(file_path, encoding="utf-8") as f:
+                for line in f:
+                    if line.strip():
+                        packages.append(json.loads(line))
 
-        push_commit = idx == len(targets_files)  # only True on last
-        store_cargo_packages(packages, fed_repo, push_commit)
-        logger(f"Processed {len(packages)} packages from {file_path} ({idx}/{len(targets_files)}).")
+            push_commit = idx == len(changed_files)
+            store_cargo_packages(packages, fed_repo, push_commit)
 
-    update_last_commit(setting_last_commit, fed_repo, "cargo")
-    logger("Updated last commit checkpoint for Cargo.")
+        update_last_commit(next_commit, fed_conf_repo, "cargo")
+        logger(f"Pushed batch for commit range {setting_last_commit}:{next_commit}.")

minecode_pipelines/pipelines/mine_cargo.py

@@ -32,6 +32,10 @@
     "FEDERATEDCODE_CARGO_GIT_URL", "https://github.com/ziadhany/cargo-test"
 )
 
+FEDERATEDCODE_CONFIG_GIT_URL = os.environ.get(
+    "FEDERATEDCODE_CONFIG_GIT_URL", "https://github.com/ziadhany/federatedcode-config"
+)
+
 
 class MineandPublishCargoPURLs(Pipeline):
     """Pipeline to mine Cargo (crates.io) packages and publish them to FederatedCode."""
@@ -50,7 +54,7 @@ def check_federatedcode_eligibility(self):
         Check if the project fulfills the following criteria for
         pushing the project result to FederatedCode.
         """
-        federatedcode.check_federatedcode_eligibility(project=self.project)
+        federatedcode.check_federatedcode_configured_and_available(project=self.project)
 
     def clone_cargo_repo(self):
         """
@@ -59,6 +63,7 @@ def clone_cargo_repo(self):
         conan_repo_url = "https://github.com/rust-lang/crates.io-index"
 
         self.fed_repo = federatedcode.clone_repository(FEDERATEDCODE_CARGO_GIT_URL)
+        self.fed_conf_repo = federatedcode.clone_repository(FEDERATEDCODE_CONFIG_GIT_URL)
         self.cargo_repo = Repo.clone_from(conan_repo_url, get_temp_file())
 
     def collect_packages_from_cargo(self):
@@ -73,3 +78,6 @@ def clean_cargo_repo(self):
 
         if self.fed_repo:
             delete_local_clone(self.fed_repo)
+
+        if self.fed_conf_repo:
+            delete_local_clone(self.fed_repo)

minecode_pipelines/pipes/__init__.py

@@ -41,6 +41,12 @@ def write_data_to_file(path, data):
         f.write(saneyaml.dump(data))
 
 
+def write_json_file(path, data):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, mode="w", encoding="utf-8") as f:
+        json.dump(data, f, indent=4, ensure_ascii=False)
+
+
 def write_purls_to_repo(repo, package, updated_purls, push_commits=False):
     """Write or update package purls in the repo and optionally commit/push changes."""
     ppath = hashid.get_package_purls_yml_file_path(package)
@@ -84,23 +90,23 @@ def get_changed_files(repo: Repo, commit_x: str = None, commit_y: str = None):
     Return a list of files changed between two commits using GitPython.
     Includes added, modified, deleted, and renamed files.
 
-    - commit_x is the empty tree hash (repo root).
-    - commit_y is the latest commit (HEAD).
+    - commit_x: base commit (or the empty tree hash for the first commit)
+    - commit_y: target commit (defaults to HEAD if not provided)
     """
+    EMPTY_TREE_HASH = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
+
     if commit_y is None:
         commit_y = repo.head.commit.hexsha
-
     commit_y_obj = repo.commit(commit_y)
-    if commit_x is None:
-        commit_x = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
 
-        diff_index = commit_y_obj.diff(commit_x, R=True)
+    if commit_x is None or commit_x == EMPTY_TREE_HASH:
+        # First commit case: diff against empty tree
+        diff_index = commit_y_obj.diff(EMPTY_TREE_HASH, R=True)
     else:
         commit_x_obj = repo.commit(commit_x)
-        diff_index = commit_x_obj.diff(commit_y_obj)
+        diff_index = commit_x_obj.diff(commit_y_obj, R=True)
 
     changed_files = {item.a_path or item.b_path for item in diff_index}
-
     return list(changed_files)
 
 
@@ -136,7 +142,7 @@ def update_last_commit(last_commit, repo, ecosystem):
     }
 
     settings_path = Path(repo.working_tree_dir) / "minecode_checkpoints" / f"{ecosystem}.json"
-    write_data_to_file(path=settings_path, data=settings_data)
+    write_json_file(path=settings_path, data=settings_data)
     repo.index.add([settings_path])
 
     commit_message = f"""\

minecode_pipelines/utils.py

@@ -6,10 +6,10 @@
 # See https://github.com/aboutcode-org/purldb for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-
 import tempfile
 import os
 from commoncode.fileutils import create_dir
+from git.repo.base import Repo
 
 
 def system_temp_dir(temp_dir=os.getenv("MINECODE_TMP")):
@@ -48,3 +48,30 @@ def get_temp_file(file_name="data", extension=".file", dir_name=""):
     temp_dir = get_temp_dir(dir_name)
     location = os.path.join(temp_dir, file_name)
     return location
+
+
+EMPTY_TREE_HASH = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
+
+
+def get_next_x_commit(repo: Repo, current_commit: str, x: int = 1, branch: str = "master") -> str:
+    if x == 0:
+        return current_commit
+
+    history = list(repo.iter_commits(branch))
+    if not history:
+        return current_commit  # no commits, return current_commit
+
+    if not current_commit or current_commit == EMPTY_TREE_HASH:
+        if x == 1:
+            return history[-1].hexsha
+        else:
+            return history[0].hexsha
+
+    for i, commit in enumerate(history):
+        if commit.hexsha == current_commit:
+            if i + x < len(history):
+                return history[i + x].hexsha
+            else:
+                return history[0].hexsha
+
+    return history[0].hexsha