Merge latest main branch

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

.github/workflows/docs.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.8]
+        python-version: [3.9]
 
     steps:
       - name: Checkout code

.github/workflows/main.yml

@@ -29,7 +29,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
 
     steps:
       - name: Checkout code

CHANGELOG.rst

@@ -1,6 +1,22 @@
 Release notes
 =============
 
+Version (next)
+-------------------
+
+- Add Pipeline to flag ghost packages (#1533)
+- Add logging configuration (#1533)
+- Drop support for python 3.8 (#1533)
+
+
+Version v34.0.0
+-------------------
+
+- Improve API performance.
+- Add severity range score in API.
+- Refactor GitlabDataSource to work with browser extension
+
+
 Version v34.0.0rc5
 -------------------
 

requirements.txt

@@ -1,3 +1,4 @@
+aboutcode.pipeline==0.1.0
 aiosignal==1.2.0
 alabaster==0.7.12
 asgiref==3.5.2
@@ -10,6 +11,7 @@ bcrypt==3.2.0
 beautifulsoup4==4.10.0
 binaryornot==0.4.4
 black==22.3.0
+bleach==6.1.0
 boolean.py==3.8
 certifi==2024.7.4
 cffi==1.15.0
@@ -49,6 +51,7 @@ jsonschema==3.2.0
 license-expression==21.6.14
 lxml==4.9.1
 Markdown==3.3.4
+markdown-it-py==3.0.0
 MarkupSafe==2.1.1
 matplotlib-inline==0.1.3
 multidict==6.0.2

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = vulnerablecode
-version = 33.6.3
+version = 34.0.0
 license = Apache-2.0 AND CC-BY-SA-4.0
 
 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390
@@ -48,7 +48,7 @@ license_files =
     README.rst
 
 [options]
-python_requires = >=3.8
+python_requires = >=3.9
 
 packages=find:
 include_package_data = true
@@ -92,6 +92,9 @@ install_requires =
     requests>=2.25.1
     fetchcode>=0.3.0
 
+    #pipeline
+    aboutcode.pipeline>=0.1.0
+
     #vulntotal
     python-dotenv
     texttable

vulnerabilities/api.py

@@ -9,10 +9,12 @@
 
 from urllib.parse import unquote
 
+from cvss.exceptions import CVSS2MalformedError
+from cvss.exceptions import CVSS3MalformedError
+from cvss.exceptions import CVSS4MalformedError
 from django.db.models import Prefetch
 from django_filters import rest_framework as filters
 from drf_spectacular.utils import extend_schema
-from drf_spectacular.utils import inline_serializer
 from packageurl import PackageURL
 from packageurl import normalize_qualifiers
 from rest_framework import serializers
@@ -32,7 +34,10 @@
 from vulnerabilities.models import VulnerabilitySeverity
 from vulnerabilities.models import Weakness
 from vulnerabilities.models import get_purl_query_lookups
+from vulnerabilities.severity_systems import EPSS
+from vulnerabilities.severity_systems import SCORING_SYSTEMS
 from vulnerabilities.throttling import StaffUserRateThrottle
+from vulnerabilities.utils import get_severity_range
 
 
 class VulnerabilitySeveritySerializer(serializers.ModelSerializer):
@@ -82,29 +87,23 @@ def get_resource_url(self, instance):
         return resource_url
 
 
-class MinimalPackageSerializer(BaseResourceSerializer):
+class VulnVulnIDSerializer(serializers.Serializer):
     """
-    Used for nesting inside vulnerability focused APIs.
+    Serializer for the series of vulnerability IDs.
     """
 
-    def get_affected_vulnerabilities(self, package):
-        parent_affected_vulnerabilities = package.fixed_package_details.get("vulnerabilities") or []
-
-        affected_vulnerabilities = [
-            self.get_vulnerability(vuln) for vuln in parent_affected_vulnerabilities
-        ]
+    vulnerability = serializers.CharField(source="vulnerability_id")
 
-        return affected_vulnerabilities
+    class Meta:
+        fields = ["vulnerability"]
 
-    def get_vulnerability(self, vuln):
-        affected_vulnerability = {}
 
-        vulnerability = vuln.get("vulnerability")
-        if vulnerability:
-            affected_vulnerability["vulnerability"] = vulnerability.vulnerability_id
-            return affected_vulnerability
+class MinimalPackageSerializer(BaseResourceSerializer):
+    """
+    Used for nesting inside vulnerability focused APIs.
+    """
 
-    affected_by_vulnerabilities = serializers.SerializerMethodField("get_affected_vulnerabilities")
+    affected_by_vulnerabilities = VulnVulnIDSerializer(source="affecting_vulns", many=True)
 
     purl = serializers.CharField(source="package_url")
 
@@ -140,18 +139,17 @@ class VulnSerializerRefsAndSummary(BaseResourceSerializer):
     Lookup vulnerabilities references by aliases (such as a CVE).
     """
 
-    def to_representation(self, instance):
-        data = super().to_representation(instance)
-        aliases = [alias["alias"] for alias in data["aliases"]]
-        data["aliases"] = aliases
-        return data
-
     fixed_packages = MinimalPackageSerializer(
         many=True, source="filtered_fixed_packages", read_only=True
     )
 
     references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set")
-    aliases = AliasSerializer(many=True, source="alias")
+
+    aliases = serializers.SerializerMethodField()
+
+    def get_aliases(self, obj):
+        # Assuming `obj.aliases` is a queryset of `Alias` objects
+        return [alias.alias for alias in obj.aliases.all()]
 
     class Meta:
         model = Vulnerability
@@ -193,6 +191,7 @@ class VulnerabilitySerializer(BaseResourceSerializer):
     aliases = AliasSerializer(many=True, source="alias")
     kev = KEVSerializer(read_only=True)
     weaknesses = WeaknessSerializer(many=True)
+    severity_range_score = serializers.SerializerMethodField()
 
     def to_representation(self, instance):
         data = super().to_representation(instance)
@@ -206,6 +205,30 @@ def to_representation(self, instance):
 
         return data
 
+    def get_severity_range_score(self, instance):
+        severity_vectors = []
+        severity_values = set()
+        for s in instance.severities:
+            if s.scoring_system == EPSS.identifier:
+                continue
+
+            if s.scoring_elements and s.scoring_system in SCORING_SYSTEMS:
+                try:
+                    vector_values = SCORING_SYSTEMS[s.scoring_system].get(s.scoring_elements)
+                    severity_vectors.append(vector_values)
+                except (
+                    CVSS2MalformedError,
+                    CVSS3MalformedError,
+                    CVSS4MalformedError,
+                    NotImplementedError,
+                ):
+                    pass
+
+            if s.value:
+                severity_values.add(s.value)
+        severity_range = get_severity_range(severity_values)
+        return severity_range
+
     class Meta:
         model = Vulnerability
         fields = [
@@ -218,6 +241,7 @@ class Meta:
             "references",
             "weaknesses",
             "kev",
+            "severity_range_score",
         ]
 
 
@@ -226,34 +250,22 @@ class PackageSerializer(BaseResourceSerializer):
     Lookup software package using Package URLs
     """
 
-    def to_representation(self, instance):
-        data = super().to_representation(instance)
-        data["qualifiers"] = normalize_qualifiers(data["qualifiers"], encode=False)
-
-        return data
-
-    next_non_vulnerable_version = serializers.SerializerMethodField("get_next_non_vulnerable")
-
-    def get_next_non_vulnerable(self, package):
-        next_non_vulnerable = package.fixed_package_details.get("next_non_vulnerable", None)
-        if next_non_vulnerable:
-            return next_non_vulnerable.version
-
-    latest_non_vulnerable_version = serializers.SerializerMethodField("get_latest_non_vulnerable")
-
-    def get_latest_non_vulnerable(self, package):
-        latest_non_vulnerable = package.fixed_package_details.get("latest_non_vulnerable", None)
-        if latest_non_vulnerable:
-            return latest_non_vulnerable.version
+    next_non_vulnerable_version = serializers.CharField(read_only=True)
+    latest_non_vulnerable_version = serializers.CharField(read_only=True)
 
     purl = serializers.CharField(source="package_url")
 
     affected_by_vulnerabilities = serializers.SerializerMethodField("get_affected_vulnerabilities")
 
     fixing_vulnerabilities = serializers.SerializerMethodField("get_fixing_vulnerabilities")
 
+    qualifiers = serializers.SerializerMethodField()
+
     is_vulnerable = serializers.BooleanField()
 
+    def get_qualifiers(self, package):
+        return normalize_qualifiers(package.qualifiers, encode=False)
+
     def get_fixed_packages(self, package):
         """
         Return a queryset of all packages that fix a vulnerability with
@@ -337,8 +349,6 @@ class Meta:
             "fixing_vulnerabilities",
         ]
 
-    is_vulnerable = serializers.BooleanField()
-
 
 class PackageFilterSet(filters.FilterSet):
     purl = filters.CharFilter(method="filter_purl")

vulnerabilities/import_runner.py

@@ -18,7 +18,6 @@
 
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import Importer
-from vulnerabilities.importers import IMPORTERS_REGISTRY
 from vulnerabilities.improver import Inference
 from vulnerabilities.improvers.default import DefaultImporter
 from vulnerabilities.models import Advisory

vulnerabilities/importers/__init__.py

@@ -30,7 +30,6 @@
 from vulnerabilities.importers import oss_fuzz
 from vulnerabilities.importers import postgresql
 from vulnerabilities.importers import project_kb_msr2019
-from vulnerabilities.importers import pypa
 from vulnerabilities.importers import pysec
 from vulnerabilities.importers import redhat
 from vulnerabilities.importers import retiredotnet
@@ -40,13 +39,13 @@
 from vulnerabilities.importers import ubuntu_usn
 from vulnerabilities.importers import vulnrichment
 from vulnerabilities.importers import xen
+from vulnerabilities.pipelines import pypa_importer
 
 IMPORTERS_REGISTRY = [
     nvd.NVDImporter,
     github.GitHubAPIImporter,
     gitlab.GitLabAPIImporter,
     npm.NpmImporter,
-    pypa.PyPaImporter,
     nginx.NginxImporter,
     pysec.PyPIImporter,
     alpine_linux.AlpineImporter,
@@ -75,6 +74,7 @@
     github_osv.GithubOSVImporter,
     epss.EPSSImporter,
     vulnrichment.VulnrichImporter,
+    pypa_importer.PyPaImporterPipeline,
 ]
 
 IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

vulnerabilities/importers/pypa.py

@@ -10,6 +10,7 @@
 from vulnerabilities.improvers import valid_versions
 from vulnerabilities.improvers import vulnerability_kev
 from vulnerabilities.improvers import vulnerability_status
+from vulnerabilities.pipelines import flag_ghost_packages
 
 IMPROVERS_REGISTRY = [
     valid_versions.GitHubBasicImprover,
@@ -29,6 +30,7 @@
     valid_versions.GithubOSVImprover,
     vulnerability_status.VulnerabilityStatusImprover,
     vulnerability_kev.VulnerabilityKevImprover,
+    flag_ghost_packages.FlagGhostPackagePipeline,
 ]
 
 IMPROVERS_REGISTRY = {x.qualified_name: x for x in IMPROVERS_REGISTRY}