Use last incremental from index properties #660

Signed-off-by: Jono Yang <[email protected]>

Author: JonoYang

minecode_pipelines/pipes/__init__.py

@@ -7,12 +7,20 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import json
 import os
+import requests
 import saneyaml
 
+from datetime import datetime
 from pathlib import Path
 
 from aboutcode.hashid import PURLS_FILENAME
+from scanpipe.pipes.federatedcode import clone_repository
+from scanpipe.pipes.federatedcode import commit_and_push_changes
+
+
+MINECODE_SETTINGS_REPO = "https://github.com/AyanSinhaMahapatra/minecode-test/"
 
 
 def write_packageurls_to_file(repo, base_dir, packageurls):
@@ -26,3 +34,41 @@ def write_data_to_file(path, data):
     path.parent.mkdir(parents=True, exist_ok=True)
     with open(path, encoding="utf-8", mode="w") as f:
         f.write(saneyaml.dump(data))
+
+
+def fetch_last_serial_mined(
+    settings_repo=MINECODE_SETTINGS_REPO,
+    settings_path=None,
+):
+    """
+    Fetch "last_serial" for the last mined packages.
+
+    This is a simple JSON in a github repo containing mining checkpoints
+    with the "last_serial" from the pypi index which was mined. Example:
+    https://github.com/AyanSinhaMahapatra/minecode-test/blob/main/minecode_checkpoints/pypi.json
+    """
+    repo_name = settings_repo.split("github.com")[-1]
+    minecode_checkpoint_pypi = (
+        "https://raw.githubusercontent.com/" + repo_name + "refs/heads/main/" + settings_path
+    )
+    response = requests.get(minecode_checkpoint_pypi)
+    if not response.ok:
+        return
+
+    settings_data = json.loads(response.text)
+    return settings_data.get("last_serial")
+
+
+def update_last_serial_mined(
+    last_serial,
+    settings_repo=MINECODE_SETTINGS_REPO,
+    settings_path=None,
+):
+    settings_data = {
+        "date": str(datetime.now()),
+        "last_serial": last_serial,
+    }
+    cloned_repo = clone_repository(repo_url=settings_repo)
+    settings_path = os.path.join(cloned_repo.working_dir, settings_path)
+    write_data_to_file(path=settings_path, data=settings_data)
+    commit_and_push_changes(repo=cloned_repo, file_to_commit=settings_path)

minecode_pipelines/pipes/maven.py

@@ -7,26 +7,29 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+from collections import namedtuple
 from itertools import chain
 import os
 import gzip
 import io
 import logging
-import arrow
-import javaproperties
-from packageurl import PackageURL
 
 from dateutil import tz
-from minecode_pipelines.pipes import java_stream
-from collections import namedtuple
-from scanpipe.pipes.fetch import fetch_http
-from scanpipe.pipes import federatedcode
-from minecode_pipeline.pipes import write_purls_to_repo
 from jawa.util.utf import decode_modified_utf8
-from packagedcode.maven import get_urls
+import arrow
+import javaproperties
+
 from packagedcode.maven import build_filename
 from packagedcode.maven import build_url
+from packagedcode.maven import get_urls
 from packagedcode.models import PackageData
+from packageurl import PackageURL
+from scanpipe.pipes.fetch import fetch_http
+from scanpipe.pipes import federatedcode
+
+from minecode_pipelines import miners
+from minecode_pipelines import pipes
+from minecode_pipelines.pipes import java_stream
 
 
 logger = logging.getLogger(__name__)
@@ -44,8 +47,9 @@
 
 MAVEN_BASE_URL = "https://repo1.maven.org/maven2"
 MAVEN_INDEX_URL = "https://repo1.maven.org/maven2/.index/nexus-maven-repository-index.gz"
-MAVEN_INDEX_INCREMENT_BASE_URL = f"https://repo1.maven.org/maven2/.index/nexus-maven-repository-index.{index}.gz"
+MAVEN_INDEX_INCREMENT_BASE_URL = "https://repo1.maven.org/maven2/.index/nexus-maven-repository-index.{index}.gz"
 MAVEN_INDEX_PROPERTIES_URL = "https://repo1.maven.org/maven2/.index/nexus-maven-repository-index.properties"
+MAVEN_SETTINGS_PATH = "minecode_checkpoints/maven.json"
 
 
 def is_worthy_artifact(artifact):
@@ -577,6 +581,14 @@ class MavenNexusCollector:
     WARNING: Processing is rather long: a full index is ~600MB.
     """
 
+    def __init__(self, index_properties_location=None):
+        if index_properties_location:
+            content = index_properties_location
+        else:
+            content = self.fetch_index_properties()
+        with open(content) as config_file:
+            self.index_properties = javaproperties.load(config_file) or {}
+
     def fetch_index(self, uri=MAVEN_INDEX_URL):
         """
         Return a temporary location where the maven index was saved.
@@ -595,11 +607,11 @@ def fetch_index_increments(self):
         """
         Yield maven index increments
         """
-        content = self.fetch_index_properties()
-        with open(content) as config_file:
-            properties = javaproperties.load(config_file) or {}
-
-        for key, increment_index in properties.items():
+        # in this context, last serial means last incremental
+        last_incremental = pipes.fetch_last_serial_mined(settings_path=MAVEN_SETTINGS_PATH)
+        for key, increment_index in self.index_properties.items():
+            if increment_index <= last_incremental:
+                continue
             if key.startswith("nexus.index.incremental"):
                 index_increment_url = MAVEN_INDEX_INCREMENT_BASE_URL.format(index=increment_index)
                 index_increment = fetch_http(index_increment_url)
@@ -676,9 +688,10 @@ def _get_packages_from_index_increments(self):
         for index_increment in self.fetch_index_increments():
             return self._get_packages(content=index_increment)
 
-    def get_packages(self, content=None, increments=False):
+    def get_packages(self, content=None):
         """Yield Package objects from maven index"""
-        if increments:
+        last_incremental = pipes.fetch_last_serial_mined(settings_path=MAVEN_SETTINGS_PATH)
+        if last_incremental:
             packages = chain(self._get_packages_from_index_increments())
         else:
             if content:
@@ -698,7 +711,6 @@ def collect_packages_from_maven(commits_per_push=10, logger=None):
         if not prev_purl:
             prev_purl = current_purl
         elif prev_purl != current_purl:
-            # check out repo
             repo_url, _ = federatedcode.get_package_repository(
                 project_purl=prev_purl,
                 logger=logger
@@ -710,16 +722,18 @@ def collect_packages_from_maven(commits_per_push=10, logger=None):
 
             push_commit = not bool(i % commits_per_push)
             # save purls to yaml
-            write_purls_to_repo(
+            miners.write_purls_to_repo(
                 repo=repo,
                 package=prev_purl,
                 packages=current_packages,
                 push_commit=push_commit
             )
 
-            # delete local clone
             federatedcode.delete_local_clone(repo)
 
             current_packages = []
             prev_purl = current_purl
         current_packages.append(package)
+
+        last_incremental = maven_nexus_collector.index_properties.get("nexus.index.last-incremental")
+        pipes.update_last_serial_mined(last_serial=last_incremental, settings_path=MAVEN_SETTINGS_PATH)

minecode_pipelines/pipes/pypi.py

@@ -20,80 +20,30 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-import os
-import json
-import requests
-
-from datetime import datetime
-
 from minecode_pipelines import pipes
 from minecode_pipelines.miners.pypi import get_pypi_packages
-from minecode_pipelines.miners.pypi import load_pypi_packages
 from minecode_pipelines.miners.pypi import get_pypi_packageurls
+from minecode_pipelines.miners.pypi import load_pypi_packages
 from minecode_pipelines.miners.pypi import PYPI_REPO
-
 from minecode_pipelines.miners.pypi import PYPI_TYPE
 
-from packageurl import PackageURL
-
 from aboutcode.hashid import get_package_base_dir
-
-
+from packageurl import PackageURL
 from scanpipe.pipes.federatedcode import clone_repository
 from scanpipe.pipes.federatedcode import commit_changes
 from scanpipe.pipes.federatedcode import push_changes
-from scanpipe.pipes.federatedcode import commit_and_push_changes
 
 
-MINECODE_SETTINGS_REPO = "https://github.com/AyanSinhaMahapatra/minecode-test/"
 PYPI_SETTINGS_PATH = "minecode_checkpoints/pypi.json"
 
 
 def mine_pypi_packages(logger=None):
     return get_pypi_packages(pypi_repo=PYPI_REPO, logger=logger)
 
 
-def fetch_last_serial_mined(
-    settings_repo=MINECODE_SETTINGS_REPO,
-    settings_path=PYPI_SETTINGS_PATH,
-):
-    """
-    Fetch "last_serial" for the last mined packages.
-
-    This is a simple JSON in a github repo containing mining checkpoints
-    with the "last_serial" from the pypi index which was mined. Example:
-    https://github.com/AyanSinhaMahapatra/minecode-test/blob/main/minecode_checkpoints/pypi.json
-    """
-    repo_name = settings_repo.split("github.com")[-1]
-    minecode_checkpoint_pypi = (
-        "https://raw.githubusercontent.com/" + repo_name + "refs/heads/main/" + settings_path
-    )
-    response = requests.get(minecode_checkpoint_pypi)
-    if not response.ok:
-        return
-
-    settings_data = json.loads(response.text)
-    return settings_data.get("last_serial")
-
-
-def update_last_serial_mined(
-    last_serial,
-    settings_repo=MINECODE_SETTINGS_REPO,
-    settings_path=PYPI_SETTINGS_PATH,
-):
-    settings_data = {
-        "date": str(datetime.now()),
-        "last_serial": last_serial,
-    }
-    cloned_repo = clone_repository(repo_url=settings_repo)
-    settings_path = os.path.join(cloned_repo.working_dir, settings_path)
-    pipes.write_data_to_file(path=settings_path, data=settings_data)
-    commit_and_push_changes(repo=cloned_repo, file_to_commit=settings_path)
-
-
 def mine_and_publish_pypi_packageurls(packages, use_last_serial=False, logger=None):
     if use_last_serial:
-        last_serial_fetched = fetch_last_serial_mined()
+        last_serial_fetched = pipes.fetch_last_serial_mined(settings_path=PYPI_SETTINGS_PATH)
         if logger:
             logger(f"Last serial number mined: {last_serial_fetched}")
 
@@ -108,9 +58,9 @@ def mine_and_publish_pypi_packageurls(packages, use_last_serial=False, logger=No
 
     if packages:
         # clone repo
-        cloned_repo = clone_repository(repo_url=MINECODE_SETTINGS_REPO)
+        cloned_repo = clone_repository(repo_url=pipes.MINECODE_SETTINGS_REPO)
         if logger:
-            logger(f"{MINECODE_SETTINGS_REPO} repo cloned at: {cloned_repo.working_dir}")
+            logger(f"{pipes.MINECODE_SETTINGS_REPO} repo cloned at: {cloned_repo.working_dir}")
 
     purl_files_updated = []
     for package in packages:
@@ -155,4 +105,4 @@ def mine_and_publish_pypi_packageurls(packages, use_last_serial=False, logger=No
 
     # update last_serial to minecode checkpoints
     if use_last_serial:
-        update_last_serial_mined(last_serial=last_serial)
+        pipes.update_last_serial_mined(last_serial=last_serial)