Mine only the unprocessed data.

Add logging and fix bug in process_cargo_packages

Signed-off-by: ziad hany <[email protected]>

Author: ziadhany

minecode_pipelines/miners/cargo.py

@@ -6,37 +6,47 @@
 # See https://github.com/aboutcode-org/purldb for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
+from minecode_pipelines.pipes import get_last_commit, get_changed_files, update_last_commit
+from minecode_pipelines.pipes.cargo import store_cargo_packages
 import json
 from pathlib import Path
 
-from minecode_pipelines.pipes.cargo import store_cargo_packages
-from minecode_pipelines.utils import get_changed_files
-
 
-def process_cargo_packages(cargo_repo, fed_repo):
+def process_cargo_packages(cargo_repo, fed_repo, logger):
     base_path = Path(cargo_repo.working_tree_dir)
-    valid_files = get_changed_files(cargo_repo)  # start from empty tree hash
+    setting_last_commit = get_last_commit(fed_repo, "cargo")
+    valid_files = get_changed_files(cargo_repo, setting_last_commit)  # start from empty tree hash
 
-    json_files = []
+    logger(f"Found {len(valid_files)} changed files in Cargo index.")
+    targets_files = []
     for file_path in base_path.glob("**/*"):
-        if not file_path.is_file() or file_path not in valid_files:
+        if not file_path.is_file():
+            continue
+
+        rel_path = str(file_path.relative_to(base_path))
+        if rel_path not in valid_files:
             continue
 
         if file_path.name in {"config.json", "README.md", "update-dl-url.yml"}:
             continue
-        json_files.append(file_path)
 
-    for idx, file_path in enumerate(json_files, start=1):
-        try:
-            with open(file_path, encoding="utf-8") as f:
-                packages = []
-                for line in f:
-                    if line.strip():
-                        packages.append(json.loads(line))
+        targets_files.append(file_path)
+
+    logger(f"Collected {len(targets_files)} target package files to process.")
 
-        except (json.JSONDecodeError, UnicodeDecodeError):
+    for idx, file_path in enumerate(targets_files, start=1):
+        packages = []
+        with open(file_path, encoding="utf-8") as f:
+            for line in f:
+                if line.strip():
+                    packages.append(json.loads(line))
+
+        if not packages:
             continue
 
-        if packages:
-            push_commit = idx == len(json_files)  # only True on last
-            store_cargo_packages(packages, fed_repo, push_commit)
+        push_commit = idx == len(targets_files)  # only True on last
+        store_cargo_packages(packages, fed_repo, push_commit)
+        logger(f"Processed {len(packages)} packages from {file_path} ({idx}/{len(targets_files)}).")
+
+    update_last_commit(setting_last_commit, fed_repo, "cargo")
+    logger("Updated last commit checkpoint for Cargo.")

minecode_pipelines/pipelines/mine_cargo.py

@@ -19,14 +19,19 @@
 #
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
+import os
 
 from git.repo.base import Repo
 from scanpipe.pipes.federatedcode import delete_local_clone
-from minecode.utils import get_temp_file
+from minecode_pipelines.utils import get_temp_file
 from scanpipe.pipelines import Pipeline
 from scanpipe.pipes import federatedcode
 from minecode_pipelines.miners import cargo
 
+FEDERATEDCODE_CARGO_GIT_URL = os.environ.get(
+    "FEDERATEDCODE_CARGO_GIT_URL", "https://github.com/ziadhany/cargo-test"
+)
+
 
 class MineandPublishCargoPURLs(Pipeline):
     """Pipeline to mine Cargo (crates.io) packages and publish them to FederatedCode."""
@@ -49,16 +54,15 @@ def check_federatedcode_eligibility(self):
 
     def clone_cargo_repo(self):
         """
-        Clone the repo at repo_url and return the VCSResponse object
+        Clone the repo at repo_url and return the Repo object
         """
-        conan_repo_url = "git+https://github.com/rust-lang/crates.io-index"
-        fed_repo_url = "git+https://github.com/ziadhany/cargo-test"
+        conan_repo_url = "https://github.com/rust-lang/crates.io-index"
 
-        self.fed_repo = federatedcode.clone_repository(fed_repo_url)
+        self.fed_repo = federatedcode.clone_repository(FEDERATEDCODE_CARGO_GIT_URL)
         self.cargo_repo = Repo.clone_from(conan_repo_url, get_temp_file())
 
     def collect_packages_from_cargo(self):
-        cargo.process_cargo_packages(self.cargo_repo, self.fed_repo)
+        cargo.process_cargo_packages(self.cargo_repo, self.fed_repo, self.log)
 
     def clean_cargo_repo(self):
         """

minecode_pipelines/pipes/__init__.py

@@ -8,11 +8,18 @@
 #
 
 from aboutcode.hashid import PURLS_FILENAME
-import os
 import textwrap
-from pathlib import Path
 import saneyaml
 from aboutcode import hashid
+import os
+from datetime import datetime
+from pathlib import Path
+
+from git import Repo
+import json
+
+from minecode_pipelines.miners import write_data_to_file
+
 
 VERSION = os.environ.get("VERSION", "")
 PURLDB_ALLOWED_HOST = os.environ.get("FEDERATEDCODE_GIT_ALLOWED_HOST", "")
@@ -70,3 +77,76 @@ def commit_and_push_changes(repo):
     default_branch = repo.active_branch.name
     repo.index.commit(textwrap.dedent(commit_message))
     repo.git.push(remote_name, default_branch, "--no-verify")
+
+
+def get_changed_files(repo: Repo, commit_x: str = None, commit_y: str = None):
+    """
+    Return a list of files changed between two commits using GitPython.
+    Includes added, modified, deleted, and renamed files.
+
+    - commit_x is the empty tree hash (repo root).
+    - commit_y is the latest commit (HEAD).
+    """
+    if commit_y is None:
+        commit_y = repo.head.commit.hexsha
+
+    commit_y_obj = repo.commit(commit_y)
+    if commit_x is None:
+        commit_x = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
+
+        diff_index = commit_y_obj.diff(commit_x, R=True)
+    else:
+        commit_x_obj = repo.commit(commit_x)
+        diff_index = commit_x_obj.diff(commit_y_obj)
+
+    changed_files = {item.a_path or item.b_path for item in diff_index}
+
+    return list(changed_files)
+
+
+def get_last_commit(repo, ecosystem):
+    """
+    Retrieve the last mined commit for a given ecosystem.
+
+    This function reads a JSON checkpoint file from the repository, which stores
+    mining progress. Each checkpoint contains the "last_commit" from the package
+    index (e.g., PyPI) that was previously mined.
+
+    https://github.com/AyanSinhaMahapatra/minecode-test/blob/main/minecode_checkpoints/pypi.json
+    https://github.com/ziadhany/cargo-test/blob/main/minecode_checkpoints/cargo.json
+    """
+
+    last_commit_file_path = (
+        Path(repo.working_tree_dir) / "minecode_checkpoints" / f"{ecosystem}.json"
+    )
+    try:
+        with open(last_commit_file_path) as f:
+            settings_data = json.load(f)
+    except FileNotFoundError:
+        return
+    return settings_data.get("last_commit")
+
+
+def update_last_commit(last_commit, repo, ecosystem):
+    """Update the last mined commit checkpoint for a given ecosystem and push it to the repo."""
+
+    settings_data = {
+        "date": str(datetime.now()),
+        "last_commit": last_commit,
+    }
+
+    settings_path = Path(repo.working_tree_dir) / "minecode_checkpoints" / f"{ecosystem}.json"
+    write_data_to_file(path=settings_path, data=settings_data)
+    repo.index.add([settings_path])
+
+    commit_message = f"""\
+    Update last mined commit for {ecosystem}
+
+    Tool: pkg:github/aboutcode-org/purldb@v{VERSION}
+    Reference: https://{PURLDB_ALLOWED_HOST}/
+    Signed-off-by: {author_name} <{author_email}>
+    """
+
+    default_branch = repo.active_branch.name
+    repo.index.commit(textwrap.dedent(commit_message))
+    repo.git.push(remote_name, default_branch, "--no-verify")

minecode_pipelines/utils.py

@@ -10,7 +10,6 @@
 import tempfile
 import os
 from commoncode.fileutils import create_dir
-from git import Repo
 
 
 def system_temp_dir(temp_dir=os.getenv("MINECODE_TMP")):
@@ -49,26 +48,3 @@ def get_temp_file(file_name="data", extension=".file", dir_name=""):
     temp_dir = get_temp_dir(dir_name)
     location = os.path.join(temp_dir, file_name)
     return location
-
-
-def get_changed_files(
-    repo: Repo, commit_x: str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904", commit_y: str = None
-):
-    """
-    Return a list of files changed between two commits using GitPython.
-    Includes added, modified, deleted, and renamed files.
-
-    - commit_x is the empty tree hash (repo root).
-    - commit_y is the latest commit (HEAD).
-    """
-
-    if commit_y is None:
-        commit_y = repo.head.commit.hexsha
-
-    commit_x_obj = repo.commit(commit_x)
-    commit_y_obj = repo.commit(commit_y)
-
-    diff_index = commit_x_obj.diff(commit_y_obj)
-    changed_files = {item.a_path or item.b_path for item in diff_index}
-
-    return list(changed_files)