Support mining pypi purls with checkpoints (#708)

Reference: #662

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

minecode_pipelines/__init__.py

@@ -6,3 +6,5 @@
 # See https://github.com/aboutcode-org/purldb for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
+
+VERSION = "0.0.1b2"

minecode_pipelines/miners/pypi.py

@@ -14,6 +14,7 @@
 from packageurl import PackageURL
 
 from minecode_pipelines.utils import get_temp_file
+from minecode_pipelines.pipes import write_data_to_json_file
 
 """
 Visitors for Pypi and Pypi-like Python package repositories.
@@ -48,11 +49,12 @@ def get_pypi_packages(pypi_repo, logger=None):
     if not response.ok:
         return
 
-    packages = response.json()
-    temp_file = get_temp_file("PypiPackagesJSON")
-    with open(temp_file, "w", encoding="utf-8") as f:
-        json.dump(packages, f, indent=4)
+    return response.json()
 
+
+def write_packages_json(packages, name):
+    temp_file = get_temp_file(name)
+    write_data_to_json_file(path=temp_file, data=packages)
     return temp_file
 
 
@@ -76,11 +78,19 @@ def get_pypi_packageurls(name):
     return packageurls
 
 
-def load_pypi_packages(packages):
-    with open(packages) as f:
+def load_pypi_packages(packages_file):
+    with open(packages_file) as f:
         packages_data = json.load(f)
 
     last_serial = packages_data.get("meta").get("_last-serial")
     packages = packages_data.get("projects")
 
     return last_serial, packages
+
+
+def get_last_serial_from_packages(packages_file):
+    with open(packages_file) as f:
+        packages_data = json.load(f)
+
+    last_serial = packages_data.get("meta").get("_last-serial")
+    return last_serial

minecode_pipelines/pipelines/mine_pypi.py

@@ -24,6 +24,7 @@
 from scanpipe.pipes import federatedcode
 
 from minecode_pipelines.pipes import pypi
+from minecode_pipelines import pipes
 
 
 class MineandPublishPypiPURLs(Pipeline):
@@ -38,19 +39,27 @@ def steps(cls):
             cls.check_federatedcode_eligibility,
             cls.mine_pypi_packages,
             cls.mine_and_publish_pypi_packageurls,
+            cls.delete_cloned_repos,
         )
 
     def check_federatedcode_eligibility(self):
         """
         Check if the project fulfills the following criteria for
         pushing the project result to FederatedCode.
         """
-        federatedcode.check_federatedcode_configured_and_available()
+        federatedcode.check_federatedcode_configured_and_available(logger=self.log)
 
     def mine_pypi_packages(self):
-        """Mine pypi package names from pypi indexes."""
-        self.pypi_packages = pypi.mine_pypi_packages(logger=self.log)
+        """Mine pypi package names from pypi indexes or checkpoint."""
+        self.pypi_packages, self.state = pypi.mine_pypi_packages(logger=self.log)
 
     def mine_and_publish_pypi_packageurls(self):
         """Get pypi packageURLs for all mined pypi package names."""
-        pypi.mine_and_publish_pypi_packageurls(packages=self.pypi_packages, logger=self.log)
+        self.repos = pypi.mine_and_publish_pypi_packageurls(
+            packages_file=self.pypi_packages,
+            state=self.state,
+            logger=self.log,
+        )
+
+    def delete_cloned_repos(self):
+        pipes.delete_cloned_repos(repos=self.repos, logger=self.log)

minecode_pipelines/pipes/__init__.py

@@ -8,21 +8,101 @@
 #
 
 import os
+import json
+import requests
 import saneyaml
 
 from pathlib import Path
 
 from aboutcode.hashid import PURLS_FILENAME
 
+from scanpipe.pipes.federatedcode import delete_local_clone
+from scanpipe.pipes.federatedcode import commit_and_push_changes
+
+# states:
+# note: a state is null when mining starts
+INITIAL_SYNC_STATE = "initial-sync"
+PERIODIC_SYNC_STATE = "periodic-sync"
+
+
+MINECODE_PIPELINES_CONFIG_REPO = "https://github.com/aboutcode-data/minecode-pipelines-config/"
+
+
+def fetch_checkpoint_from_github(config_repo, checkpoint_path):
+    repo_name = config_repo.split("github.com")[-1]
+    checkpoints_file = (
+        "https://raw.githubusercontent.com/" + repo_name + "refs/heads/main/" + checkpoint_path
+    )
+    response = requests.get(checkpoints_file)
+    if not response.ok:
+        return
+
+    checkpoint_data = json.loads(response.text)
+    return checkpoint_data
+
+
+def update_checkpoints_in_github(checkpoint, cloned_repo, path):
+    checkpoint_path = os.path.join(cloned_repo.working_dir, path)
+    write_data_to_json_file(path=checkpoint_path, data=checkpoint)
+    commit_message = """Update federatedcode purl mining checkpoint"""
+    commit_and_push_changes(
+        repo=cloned_repo,
+        files_to_commit=[checkpoint_path],
+        commit_message=commit_message,
+    )
+
+
+def get_mined_packages_from_checkpoint(config_repo, checkpoint_path):
+    checkpoint = fetch_checkpoint_from_github(
+        config_repo=config_repo,
+        checkpoint_path=checkpoint_path,
+    )
+    return checkpoint.get("packages_mined", [])
+
+
+def update_mined_packages_in_checkpoint(packages, config_repo, cloned_repo, checkpoint_path):
+    mined_packages = get_mined_packages_from_checkpoint(
+        config_repo=config_repo,
+        checkpoint_path=checkpoint_path,
+    )
+    packages = {"packages_mined": packages + mined_packages}
+    update_checkpoints_in_github(
+        checkpoint=packages,
+        cloned_repo=cloned_repo,
+        path=checkpoint_path,
+    )
+
 
 def write_packageurls_to_file(repo, base_dir, packageurls):
     purl_file_rel_path = os.path.join(base_dir, PURLS_FILENAME)
     purl_file_full_path = Path(repo.working_dir) / purl_file_rel_path
-    write_data_to_file(path=purl_file_full_path, data=packageurls)
+    write_data_to_yaml_file(path=purl_file_full_path, data=packageurls)
     return purl_file_rel_path
 
 
-def write_data_to_file(path, data):
+def write_data_to_yaml_file(path, data):
+    if isinstance(path, str):
+        path = Path(path)
+
     path.parent.mkdir(parents=True, exist_ok=True)
     with open(path, encoding="utf-8", mode="w") as f:
         f.write(saneyaml.dump(data))
+
+
+def write_data_to_json_file(path, data):
+    if isinstance(path, str):
+        path = Path(path)
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=4)
+
+
+def delete_cloned_repos(repos, logger=None):
+    if not repos:
+        return
+
+    for repo in repos:
+        if logger:
+            logger(f"Deleting local clone at: {repo.working_dir}")
+        delete_local_clone(repo)