Implement functionality to process bulk of commits

Signed-off-by: ziad hany <[email protected]>

Author: ziadhany

minecode_pipelines/miners/cargo.py

@@ -6,47 +6,61 @@
 # See https://github.com/aboutcode-org/purldb for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-from minecode_pipelines.pipes import get_last_commit, get_changed_files, update_last_commit
+from minecode_pipelines.pipes import get_last_commit
+from minecode_pipelines.pipes import get_changed_files
+from minecode_pipelines.pipes import update_last_commit
 from minecode_pipelines.pipes.cargo import store_cargo_packages
 import json
 from pathlib import Path
 
+from minecode_pipelines.utils import get_next_x_commit
+
+
+def process_cargo_packages(cargo_repo, fed_repo, fed_conf_repo, logger):
+    """
+    Process Cargo index files commit by commit.
+    Push changes to fed_repo after:
+    - every `commit_batch_size` commits, OR
+    - every `file_batch_size` files, OR
+    - when reaching HEAD.
+    """
 
-def process_cargo_packages(cargo_repo, fed_repo, logger):
     base_path = Path(cargo_repo.working_tree_dir)
-    setting_last_commit = get_last_commit(fed_repo, "cargo")
-    valid_files = get_changed_files(cargo_repo, setting_last_commit)  # start from empty tree hash
 
-    logger(f"Found {len(valid_files)} changed files in Cargo index.")
-    targets_files = []
-    for file_path in base_path.glob("**/*"):
-        if not file_path.is_file():
-            continue
+    while True:
+        setting_last_commit = get_last_commit(fed_conf_repo, "cargo")
+
+        if setting_last_commit is None:
+            setting_last_commit = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
 
-        rel_path = str(file_path.relative_to(base_path))
-        if rel_path not in valid_files:
-            continue
+        next_commit = get_next_x_commit(cargo_repo, setting_last_commit, x=1000, branch="master")
 
-        if file_path.name in {"config.json", "README.md", "update-dl-url.yml"}:
-            continue
+        if next_commit == setting_last_commit:
+            logger("No new commits to mine")
+            break
 
-        targets_files.append(file_path)
+        changed_files = get_changed_files(
+            cargo_repo, commit_x=setting_last_commit, commit_y=next_commit
+        )
+        logger(f"Found {len(changed_files)} changed files in Cargo index.")
 
-    logger(f"Collected {len(targets_files)} target package files to process.")
+        for idx, rel_path in enumerate(changed_files):
+            file_path = base_path / rel_path
+            logger(f"Found {file_path}.")
 
-    for idx, file_path in enumerate(targets_files, start=1):
-        packages = []
-        with open(file_path, encoding="utf-8") as f:
-            for line in f:
-                if line.strip():
-                    packages.append(json.loads(line))
+            if not file_path.is_file():
+                continue
 
-        if not packages:
-            continue
+            if file_path.name in {"config.json", "README.md", "update-dl-url.yml"}:
+                continue
+            packages = []
+            with open(file_path, encoding="utf-8") as f:
+                for line in f:
+                    if line.strip():
+                        packages.append(json.loads(line))
 
-        push_commit = idx == len(targets_files)  # only True on last
-        store_cargo_packages(packages, fed_repo, push_commit)
-        logger(f"Processed {len(packages)} packages from {file_path} ({idx}/{len(targets_files)}).")
+            push_commit = idx == len(changed_files)
+            store_cargo_packages(packages, fed_repo, push_commit)
 
-    update_last_commit(setting_last_commit, fed_repo, "cargo")
-    logger("Updated last commit checkpoint for Cargo.")
+        update_last_commit(next_commit, fed_conf_repo, "cargo")
+        logger(f"Pushed batch for commit range {setting_last_commit}:{next_commit}.")

minecode_pipelines/pipelines/mine_cargo.py

@@ -32,6 +32,10 @@
     "FEDERATEDCODE_CARGO_GIT_URL", "https://github.com/ziadhany/cargo-test"
 )
 
+FEDERATEDCODE_CONFIG_GIT_URL = os.environ.get(
+    "FEDERATEDCODE_CONFIG_GIT_URL", "https://github.com/ziadhany/federatedcode-config"
+)
+
 
 class MineandPublishCargoPURLs(Pipeline):
     """Pipeline to mine Cargo (crates.io) packages and publish them to FederatedCode."""
@@ -50,7 +54,7 @@ def check_federatedcode_eligibility(self):
         Check if the project fulfills the following criteria for
         pushing the project result to FederatedCode.
         """
-        federatedcode.check_federatedcode_eligibility(project=self.project)
+        federatedcode.check_federatedcode_configured_and_available(project=self.project)
 
     def clone_cargo_repo(self):
         """
@@ -59,6 +63,7 @@ def clone_cargo_repo(self):
         conan_repo_url = "https://github.com/rust-lang/crates.io-index"
 
         self.fed_repo = federatedcode.clone_repository(FEDERATEDCODE_CARGO_GIT_URL)
+        self.fed_conf_repo = federatedcode.clone_repository(FEDERATEDCODE_CONFIG_GIT_URL)
         self.cargo_repo = Repo.clone_from(conan_repo_url, get_temp_file())
 
     def collect_packages_from_cargo(self):
@@ -73,3 +78,6 @@ def clean_cargo_repo(self):
 
         if self.fed_repo:
             delete_local_clone(self.fed_repo)
+
+        if self.fed_conf_repo:
+            delete_local_clone(self.fed_repo)

minecode_pipelines/utils.py

@@ -6,10 +6,10 @@
 # See https://github.com/aboutcode-org/purldb for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-
 import tempfile
 import os
 from commoncode.fileutils import create_dir
+from git.repo.base import Repo
 
 from itertools import zip_longest
 
@@ -60,3 +60,30 @@ def get_temp_file(file_name="data", extension=".file", dir_name=""):
     temp_dir = get_temp_dir(dir_name)
     location = os.path.join(temp_dir, file_name)
     return location
+
+
+EMPTY_TREE_HASH = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
+
+
+def get_next_x_commit(repo: Repo, current_commit: str, x: int = 1, branch: str = "master") -> str:
+    if x == 0:
+        return current_commit
+
+    history = list(repo.iter_commits(branch))
+    if not history:
+        return current_commit  # no commits, return current_commit
+
+    if not current_commit or current_commit == EMPTY_TREE_HASH:
+        if x == 1:
+            return history[-1].hexsha
+        else:
+            return history[0].hexsha
+
+    for i, commit in enumerate(history):
+        if commit.hexsha == current_commit:
+            if i + x < len(history):
+                return history[i + x].hexsha
+            else:
+                return history[0].hexsha
+
+    return history[0].hexsha