Refactor code and fix bugs

Signed-off-by: ziad hany <[email protected]>

Author: ziadhany

minecode_pipelines/miners/cargo.py

@@ -8,10 +8,9 @@
 #
 from datetime import datetime
 
-from minecode_pipelines.pipes import fetch_checkpoint_from_github
+from minecode_pipelines.pipes import get_checkpoint_from_file
 from minecode_pipelines.pipes import get_commit_at_distance_ahead
 from minecode_pipelines.pipes import update_checkpoints_in_github
-from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO
 from minecode_pipelines.pipes import get_changed_files
 from minecode_pipelines.pipes.cargo import store_cargo_packages
 from scanpipe.pipes.federatedcode import commit_changes
@@ -38,15 +37,19 @@ def process_cargo_packages(cargo_index_repo, cloned_data_repo, config_repo, logg
     base_path = Path(cargo_index_repo.working_tree_dir)
 
     while True:
-        cargo_checkpoints = fetch_checkpoint_from_github(
-            config_repo=MINECODE_PIPELINES_CONFIG_REPO, checkpoint_path=CARGO_CHECKPOINT_PATH
+        cargo_checkpoints = get_checkpoint_from_file(
+            cloned_repo=config_repo, path=CARGO_CHECKPOINT_PATH
         )
 
         checkpoints_last_commit = cargo_checkpoints.get("last_commit")
 
-        next_commit = get_commit_at_distance_ahead(
-            cargo_index_repo, checkpoints_last_commit, num_commits_ahead=10, branch_name="master"
-        )
+        try:
+            next_commit = get_commit_at_distance_ahead(
+                cargo_index_repo, checkpoints_last_commit, num_commits_ahead=COMMIT_BATCH_SIZE, branch_name="master"
+            )
+        except ValueError as e:
+            logger(str(e))
+            break
 
         if next_commit == checkpoints_last_commit:
             logger("No new commits to mine")
@@ -75,19 +78,25 @@ def process_cargo_packages(cargo_index_repo, cloned_data_repo, config_repo, logg
             with open(file_path, encoding="utf-8") as f:
                 for line in f:
                     if line.strip():
-                        packages.append(json.loads(line))
+                        try:
+                            packages.append(json.loads(line))
+                        except json.JSONDecodeError as e:
+                            logger(f"Skipping invalid JSON in {file_path}: {e}")
 
             file_counter += 1
 
             # Commit and push after each full batch or when processing the last file
             commit_and_push = (file_counter % PACKAGE_BATCH_SIZE == 0) or (
-                idx == len(changed_files)
+                idx == len(changed_files) - 1
             )
-            purl_file, base_purl = store_cargo_packages(packages, cloned_data_repo)
-            logger(f"writing packageURLs for package: {base_purl} at: {purl_file}")
 
-            purl_files.append(purl_file)
-            purls.append(str(base_purl))
+            result_store = store_cargo_packages(packages, cloned_data_repo)
+            if result_store:
+                purl_file, base_purl = result_store
+                logger(f"writing packageURLs for package: {base_purl} at: {purl_file}")
+
+                purl_files.append(purl_file)
+                purls.append(str(base_purl))
 
             if not commit_and_push:
                 continue

minecode_pipelines/pipes/cargo.py

@@ -1,8 +1,10 @@
+from pathlib import Path
+
 from aboutcode import hashid
 from packageurl import PackageURL
 from aboutcode.hashid import get_core_purl
 
-from minecode_pipelines.pipes import write_packageurls_to_file
+from minecode_pipelines.pipes import write_data_to_yaml_file
 
 
 def store_cargo_packages(packages, repo):
@@ -25,4 +27,6 @@ def store_cargo_packages(packages, repo):
         updated_purls.append(purl)
 
     ppath = hashid.get_package_purls_yml_file_path(base_purl)
-    return write_packageurls_to_file(repo, ppath, updated_purls), base_purl
+    purl_file_full_path = Path(repo.working_dir) / ppath
+    write_data_to_yaml_file(path=purl_file_full_path, data=updated_purls)
+    return purl_file_full_path, base_purl

minecode_pipelines/tests/pipes/test_cargo.py

@@ -13,7 +13,7 @@
 
 
 class CargoPipelineTests(TestCase):
-    @patch("minecode_pipelines.pipes.cargo.write_packageurls_to_file")
+    @patch("minecode_pipelines.pipes.cargo.write_data_to_yaml_file")
     def test_collect_packages_from_cargo_calls_write(self, mock_write):
         packages_file = DATA_DIR / "c5store"
         expected_file = DATA_DIR / "c5store-expected.yaml"