Merge pull request #1584 from aboutcode-org/github-importer-pipeline

Migrate GitHub importer to aboutcode pipeline

Author: TG1999

vulnerabilities/importers/__init__.py

@@ -19,7 +19,6 @@
 from vulnerabilities.importers import epss
 from vulnerabilities.importers import fireeye
 from vulnerabilities.importers import gentoo
-from vulnerabilities.importers import github
 from vulnerabilities.importers import github_osv
 from vulnerabilities.importers import istio
 from vulnerabilities.importers import mozilla
@@ -38,14 +37,14 @@
 from vulnerabilities.importers import vulnrichment
 from vulnerabilities.importers import xen
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline
+from vulnerabilities.pipelines import github_importer
 from vulnerabilities.pipelines import gitlab_importer
 from vulnerabilities.pipelines import nginx_importer
 from vulnerabilities.pipelines import npm_importer
 from vulnerabilities.pipelines import pypa_importer
 
 IMPORTERS_REGISTRY = [
     nvd.NVDImporter,
-    github.GitHubAPIImporter,
     pysec.PyPIImporter,
     alpine_linux.AlpineImporter,
     openssl.OpensslImporter,
@@ -78,6 +77,7 @@
     npm_importer.NpmImporterPipeline,
     nginx_importer.NginxImporterPipeline,
     gitlab_importer.GitLabImporterPipeline,
+    github_importer.GitHubAPIImporterPipeline,
 ]
 
 IMPORTERS_REGISTRY = {

vulnerabilities/improvers/valid_versions.py

@@ -31,7 +31,6 @@
 from vulnerabilities.importers.debian import DebianImporter
 from vulnerabilities.importers.debian_oval import DebianOvalImporter
 from vulnerabilities.importers.elixir_security import ElixirSecurityImporter
-from vulnerabilities.importers.github import GitHubAPIImporter
 from vulnerabilities.importers.github_osv import GithubOSVImporter
 from vulnerabilities.importers.istio import IstioImporter
 from vulnerabilities.importers.oss_fuzz import OSSFuzzImporter
@@ -42,6 +41,7 @@
 from vulnerabilities.improver import Inference
 from vulnerabilities.models import Advisory
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline
+from vulnerabilities.pipelines.github_importer import GitHubAPIImporterPipeline
 from vulnerabilities.pipelines.gitlab_importer import GitLabImporterPipeline
 from vulnerabilities.pipelines.nginx_importer import NginxImporterPipeline
 from vulnerabilities.pipelines.npm_importer import NpmImporterPipeline
@@ -371,7 +371,7 @@ class GitLabBasicImprover(ValidVersionImprover):
 
 
 class GitHubBasicImprover(ValidVersionImprover):
-    importer = GitHubAPIImporter
+    importer = GitHubAPIImporterPipeline
     ignorable_versions = frozenset(
         [
             "0.1-bulbasaur",

vulnerabilities/migrations/0067_update_github_advisory_created_by.py

@@ -0,0 +1,38 @@
+# Generated by Django 4.2.15 on 2024-09-27 14:31
+
+from django.db import migrations
+
+"""
+Update the created_by field on Advisory from the old qualified_name
+to the new pipeline_id.
+"""
+
+
+def update_created_by(apps, schema_editor):
+    from vulnerabilities.pipelines.github_importer import GitHubAPIImporterPipeline
+
+    Advisory = apps.get_model("vulnerabilities", "Advisory")
+    Advisory.objects.filter(created_by="vulnerabilities.importers.github.GitHubAPIImporter").update(
+        created_by=GitHubAPIImporterPipeline.pipeline_id
+    )
+
+
+
+def reverse_update_created_by(apps, schema_editor):
+    from vulnerabilities.pipelines.github_importer import GitHubAPIImporterPipeline
+
+    Advisory = apps.get_model("vulnerabilities", "Advisory")
+    Advisory.objects.filter(created_by=GitHubAPIImporterPipeline.pipeline_id).update(
+        created_by="vulnerabilities.importers.github.GitHubAPIImporter"
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0066_update_gitlab_advisory_created_by"),
+    ]
+
+    operations = [
+        migrations.RunPython(update_created_by, reverse_code=reverse_update_created_by),
+    ]

vulnerabilities/pipelines/__init__.py

@@ -89,7 +89,12 @@ def advisories_count(self) -> int:
 
     def collect_and_store_advisories(self):
         collected_advisory_count = 0
-        progress = LoopProgress(total_iterations=self.advisories_count(), logger=self.log)
+        estimated_advisory_count = self.advisories_count()
+
+        if estimated_advisory_count > 0:
+            self.log(f"Collecting {estimated_advisory_count:,d} advisories")
+
+        progress = LoopProgress(total_iterations=estimated_advisory_count, logger=self.log)
         for advisory in progress.iter(self.collect_advisories()):
             if _obj := insert_advisory(
                 advisory=advisory,