Update Cargo miners to push after every 1k file changes.

Fix bug in process_cargo_packages.

Signed-off-by: ziad hany <[email protected]>

Author: ziadhany

minecode_pipelines/miners/cargo.py

@@ -20,8 +20,7 @@ def process_cargo_packages(cargo_repo, fed_repo, fed_conf_repo, logger):
     """
     Process Cargo index files commit by commit.
     Push changes to fed_repo after:
-    - every `commit_batch_size` commits, OR
-    - every `file_batch_size` files, OR
+    - every `commit_batch` commits, OR
     - when reaching HEAD.
     """
 
@@ -40,6 +39,7 @@ def process_cargo_packages(cargo_repo, fed_repo, fed_conf_repo, logger):
         )
         logger(f"Found {len(changed_files)} changed files in Cargo index.")
 
+        file_counter = 0
         for idx, rel_path in enumerate(changed_files):
             file_path = base_path / rel_path
             logger(f"Found {file_path}.")
@@ -49,13 +49,15 @@ def process_cargo_packages(cargo_repo, fed_repo, fed_conf_repo, logger):
 
             if file_path.name in {"config.json", "README.md", "update-dl-url.yml"}:
                 continue
+
             packages = []
             with open(file_path, encoding="utf-8") as f:
                 for line in f:
                     if line.strip():
                         packages.append(json.loads(line))
 
-            push_commit = idx == len(changed_files)
+            file_counter += 1
+            push_commit = (file_counter % 1000 == 0) or (idx == len(changed_files))
             store_cargo_packages(packages, fed_repo, push_commit)
 
         update_last_commit(next_commit, fed_conf_repo, "cargo")

minecode_pipelines/pipelines/mine_cargo.py

@@ -67,7 +67,7 @@ def clone_cargo_repo(self):
         self.cargo_repo = Repo.clone_from(conan_repo_url, get_temp_file())
 
     def collect_packages_from_cargo(self):
-        cargo.process_cargo_packages(self.cargo_repo, self.fed_repo, self.log)
+        cargo.process_cargo_packages(self.cargo_repo, self.fed_repo, self.fed_conf_repo, self.log)
 
     def clean_cargo_repo(self):
         """

minecode_pipelines/utils.py

@@ -61,6 +61,7 @@ def get_temp_file(file_name="data", extension=".file", dir_name=""):
     location = os.path.join(temp_dir, file_name)
     return location
 
+
 def get_next_x_commit(repo: Repo, current_commit: str, x: int = 10, branch: str = "master") -> str:
     """
     Get the x-th next commit after the current commit in the specified branch.
@@ -70,4 +71,4 @@ def get_next_x_commit(repo: Repo, current_commit: str, x: int = 10, branch: str
     revs = repo.git.rev_list(f"^{current_commit}", branch).splitlines()
     if len(revs) < x:
         raise ValueError(f"Not enough commits ahead; only {len(revs)} available.")
-    return revs[-x]
+    return revs[-x]