Mine only the unprocessed data.

Add logging and fix bug in process_cargo_packages

Signed-off-by: ziad hany <[email protected]>

Author: ziadhany

minecode_pipelines/miners/cargo.py

@@ -6,37 +6,47 @@
 # See https://github.com/aboutcode-org/purldb for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
+from minecode_pipelines.pipes import get_last_commit, get_changed_files, update_last_commit
+from minecode_pipelines.pipes.cargo import store_cargo_packages
 import json
 from pathlib import Path
 
-from minecode_pipelines.pipes.cargo import store_cargo_packages
-from minecode_pipelines.utils import get_changed_files
-
 
-def process_cargo_packages(cargo_repo, fed_repo):
+def process_cargo_packages(cargo_repo, fed_repo, logger):
     base_path = Path(cargo_repo.working_tree_dir)
-    valid_files = get_changed_files(cargo_repo)  # start from empty tree hash
+    setting_last_commit = get_last_commit(fed_repo, "cargo")
+    valid_files = get_changed_files(cargo_repo, setting_last_commit)  # start from empty tree hash
 
-    json_files = []
+    logger(f"Found {len(valid_files)} changed files in Cargo index.")
+    targets_files = []
     for file_path in base_path.glob("**/*"):
-        if not file_path.is_file() or file_path not in valid_files:
+        if not file_path.is_file():
+            continue
+
+        rel_path = str(file_path.relative_to(base_path))
+        if rel_path not in valid_files:
             continue
 
         if file_path.name in {"config.json", "README.md", "update-dl-url.yml"}:
             continue
-        json_files.append(file_path)
 
-    for idx, file_path in enumerate(json_files, start=1):
-        try:
-            with open(file_path, encoding="utf-8") as f:
-                packages = []
-                for line in f:
-                    if line.strip():
-                        packages.append(json.loads(line))
+        targets_files.append(file_path)
+
+    logger(f"Collected {len(targets_files)} target package files to process.")
 
-        except (json.JSONDecodeError, UnicodeDecodeError):
+    for idx, file_path in enumerate(targets_files, start=1):
+        packages = []
+        with open(file_path, encoding="utf-8") as f:
+            for line in f:
+                if line.strip():
+                    packages.append(json.loads(line))
+
+        if not packages:
             continue
 
-        if packages:
-            push_commit = idx == len(json_files)  # only True on last
-            store_cargo_packages(packages, fed_repo, push_commit)
+        push_commit = idx == len(targets_files)  # only True on last
+        store_cargo_packages(packages, fed_repo, push_commit)
+        logger(f"Processed {len(packages)} packages from {file_path} ({idx}/{len(targets_files)}).")
+
+    update_last_commit(setting_last_commit, fed_repo, "cargo")
+    logger("Updated last commit checkpoint for Cargo.")

minecode_pipelines/pipelines/mine_cargo.py

@@ -19,14 +19,19 @@
 #
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
+import os
 
 from git.repo.base import Repo
 from scanpipe.pipes.federatedcode import delete_local_clone
-from minecode.utils import get_temp_file
+from minecode_pipelines.utils import get_temp_file
 from scanpipe.pipelines import Pipeline
 from scanpipe.pipes import federatedcode
 from minecode_pipelines.miners import cargo
 
+FEDERATEDCODE_CARGO_GIT_URL = os.environ.get(
+    "FEDERATEDCODE_CARGO_GIT_URL", "https://github.com/ziadhany/cargo-test"
+)
+
 
 class MineandPublishCargoPURLs(Pipeline):
     """Pipeline to mine Cargo (crates.io) packages and publish them to FederatedCode."""
@@ -49,16 +54,15 @@ def check_federatedcode_eligibility(self):
 
     def clone_cargo_repo(self):
         """
-        Clone the repo at repo_url and return the VCSResponse object
+        Clone the repo at repo_url and return the Repo object
         """
-        conan_repo_url = "git+https://github.com/rust-lang/crates.io-index"
-        fed_repo_url = "git+https://github.com/ziadhany/cargo-test"
+        conan_repo_url = "https://github.com/rust-lang/crates.io-index"
 
-        self.fed_repo = federatedcode.clone_repository(fed_repo_url)
+        self.fed_repo = federatedcode.clone_repository(FEDERATEDCODE_CARGO_GIT_URL)
         self.cargo_repo = Repo.clone_from(conan_repo_url, get_temp_file())
 
     def collect_packages_from_cargo(self):
-        cargo.process_cargo_packages(self.cargo_repo, self.fed_repo)
+        cargo.process_cargo_packages(self.cargo_repo, self.fed_repo, self.log)
 
     def clean_cargo_repo(self):
         """

minecode_pipelines/utils.py

@@ -10,7 +10,6 @@
 import tempfile
 import os
 from commoncode.fileutils import create_dir
-from git import Repo
 
 from itertools import zip_longest
 
@@ -61,26 +60,3 @@ def get_temp_file(file_name="data", extension=".file", dir_name=""):
     temp_dir = get_temp_dir(dir_name)
     location = os.path.join(temp_dir, file_name)
     return location
-
-
-def get_changed_files(
-    repo: Repo, commit_x: str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904", commit_y: str = None
-):
-    """
-    Return a list of files changed between two commits using GitPython.
-    Includes added, modified, deleted, and renamed files.
-
-    - commit_x is the empty tree hash (repo root).
-    - commit_y is the latest commit (HEAD).
-    """
-
-    if commit_y is None:
-        commit_y = repo.head.commit.hexsha
-
-    commit_x_obj = repo.commit(commit_x)
-    commit_y_obj = repo.commit(commit_y)
-
-    diff_index = commit_x_obj.diff(commit_y_obj)
-    changed_files = {item.a_path or item.b_path for item in diff_index}
-
-    return list(changed_files)