Merge branch 'main' into merge_api_tests

Author: TG1999

CHANGELOG.rst

@@ -2,15 +2,25 @@ Release notes
 =============
 
 
-Next release
-----------------
+Version v32.0.0rc2
+--------------------
+
+- We added migration for adding apache tomcat option in severity scoring.
+
+
+Version v32.0.0rc1
+--------------------
 
 - We re-enabled support for the mozilla vulnerabilities advisories importer.
 - We re-enabled support for the gentoo vulnerabilities advisories importer.
 - We re-enabled support for the istio vulnerabilities advisories importer.
 - We re-enabled support for the kbmsr2019 vulnerabilities advisories importer.
 - We re-enabled support for the suse score advisories importer.
 - We re-enabled support for the elixir security advisories importer.
+- We re-enabled support for the apache tomcat security advisories importer.
+- We added support for CWE.
+- We added migrations to remove corrupted advisories https://github.com/nexB/vulnerablecode/issues/1086.
+
 
 Version v31.1.1
 ---------------

requirements.txt

@@ -11,7 +11,7 @@ beautifulsoup4==4.10.0
 binaryornot==0.4.4
 black==22.3.0
 boolean.py==3.8
-certifi==2021.10.8
+certifi==2022.12.7
 cffi==1.15.0
 chardet==4.0.0
 charset-normalizer==2.0.12
@@ -36,7 +36,7 @@ executing==0.8.3
 freezegun==1.2.1
 frozenlist==1.3.0
 gitdb==4.0.9
-GitPython==3.1.27
+GitPython==3.1.30
 gunicorn==20.1.0
 idna==3.3
 imagesize==1.3.0
@@ -115,7 +115,7 @@ yarl==1.7.2
 zipp==3.8.0
 dateparser==1.1.1
 fetchcode==0.2.0
-
+cwe2==2.0.0
 drf-spectacular-sidecar==2022.10.1
 drf-spectacular==0.24.2
 coreapi==2.3.3

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = vulnerablecode
-version = 31.1.1
+version = 32.0.0rc2
 license = Apache-2.0 AND CC-BY-SA-4.0
 
 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390
@@ -84,6 +84,7 @@ install_requires =
     Markdown>=3.3.0
     dateparser>=1.1.1
     cvss>=2.4
+    cwe2>=2.0.0
 
     # networking
     GitPython>=3.1.17

vulnerabilities/import_runner.py

@@ -61,6 +61,7 @@ def process_advisories(advisory_datas: Iterable[AdvisoryData], importer_name: st
             affected_packages=[pkg.to_dict() for pkg in data.affected_packages],
             references=[ref.to_dict() for ref in data.references],
             date_published=data.date_published,
+            weaknesses=data.weaknesses,
             defaults={
                 "created_by": importer_name,
                 "date_collected": datetime.datetime.now(tz=datetime.timezone.utc),

vulnerabilities/importer.py

@@ -246,6 +246,7 @@ class AdvisoryData:
     affected_packages: List[AffectedPackage] = dataclasses.field(default_factory=list)
     references: List[Reference] = dataclasses.field(default_factory=list)
     date_published: Optional[datetime.datetime] = None
+    weaknesses: List[int] = dataclasses.field(default_factory=list)
 
     def __post_init__(self):
         if self.date_published and not self.date_published.tzinfo:
@@ -258,6 +259,7 @@ def to_dict(self):
             "affected_packages": [pkg.to_dict() for pkg in self.affected_packages],
             "references": [ref.to_dict() for ref in self.references],
             "date_published": self.date_published.isoformat() if self.date_published else None,
+            "weaknesses": self.weaknesses,
         }
 
     @classmethod
@@ -273,6 +275,7 @@ def from_dict(cls, advisory_data):
             "date_published": datetime.datetime.fromisoformat(date_published)
             if date_published
             else None,
+            "weaknesses": advisory_data["weaknesses"],
         }
         return cls(**transformed)
 

vulnerabilities/importers/__init__.py

@@ -31,6 +31,8 @@
 from vulnerabilities.importers import retiredotnet
 from vulnerabilities.importers import suse_scores
 from vulnerabilities.importers import ubuntu
+from vulnerabilities.importers import ubuntu_usn
+from vulnerabilities.importers import xen
 
 IMPORTERS_REGISTRY = [
     nginx.NginxImporter,
@@ -57,6 +59,8 @@
     suse_scores.SUSESeverityScoreImporter,
     elixir_security.ElixirSecurityImporter,
     apache_tomcat.ApacheTomcatImporter,
+    xen.XenImporter,
+    ubuntu_usn.UbuntuUSNImporter,
 ]
 
 IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

vulnerabilities/importers/apache_httpd.py

@@ -7,11 +7,17 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-import asyncio
+import logging
 import urllib
+from datetime import datetime
+from typing import Iterable
+from typing import List
+from typing import Mapping
+from typing import Optional
 
 import requests
 from bs4 import BeautifulSoup
+from django.db.models.query import QuerySet
 from packageurl import PackageURL
 from univers.version_constraint import VersionConstraint
 from univers.version_range import ApacheVersionRange
@@ -21,8 +27,20 @@
 from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
+from vulnerabilities.importer import UnMergeablePackageError
 from vulnerabilities.importer import VulnerabilitySeverity
+from vulnerabilities.improver import Improver
+from vulnerabilities.improver import Inference
+from vulnerabilities.models import Advisory
+from vulnerabilities.package_managers import GitHubTagsAPI
+from vulnerabilities.package_managers import VersionAPI
 from vulnerabilities.severity_systems import APACHE_HTTPD
+from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage
+from vulnerabilities.utils import get_affected_packages_by_patched_package
+from vulnerabilities.utils import nearest_patched_package
+from vulnerabilities.utils import resolve_version_range
+
+logger = logging.getLogger(__name__)
 
 
 class ApacheHTTPDImporter(Importer):
@@ -147,7 +165,7 @@ def fetch_links(url):
     return links
 
 
-ignore_tags = {
+IGNORE_TAGS = {
     "AGB_BEFORE_AAA_CHANGES",
     "APACHE_1_2b1",
     "APACHE_1_2b10",
@@ -209,3 +227,80 @@ def fetch_links(url):
     "post_ajp_proxy",
     "pre_ajp_proxy",
 }
+
+
+class ApacheHTTPDImprover(Improver):
+    def __init__(self) -> None:
+        self.versions_fetcher_by_purl: Mapping[str, VersionAPI] = {}
+        self.vesions_by_purl = {}
+
+    @property
+    def interesting_advisories(self) -> QuerySet:
+        return Advisory.objects.filter(created_by=ApacheHTTPDImporter.qualified_name)
+
+    def get_package_versions(
+        self, package_url: PackageURL, until: Optional[datetime] = None
+    ) -> List[str]:
+        """
+        Return a list of `valid_versions` for the `package_url`
+        """
+        api_name = "apache/httpd"
+        versions_fetcher = GitHubTagsAPI()
+        return versions_fetcher.get_until(package_name=api_name, until=until).valid_versions
+
+    def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]:
+        """
+        Yield Inferences for the given advisory data
+        """
+        if not advisory_data.affected_packages:
+            return
+        try:
+            purl, affected_version_ranges, _ = AffectedPackage.merge(
+                advisory_data.affected_packages
+            )
+        except UnMergeablePackageError:
+            logger.error(f"Cannot merge with different purls {advisory_data.affected_packages!r}")
+            return iter([])
+
+        pkg_type = purl.type
+        pkg_namespace = purl.namespace
+        pkg_name = purl.name
+
+        if not self.vesions_by_purl.get(str(purl)):
+            valid_versions = self.get_package_versions(
+                package_url=purl, until=advisory_data.date_published
+            )
+            self.vesions_by_purl[str(purl)] = valid_versions
+
+        valid_versions = self.vesions_by_purl[str(purl)]
+
+        for affected_version_range in affected_version_ranges:
+            aff_vers, unaff_vers = resolve_version_range(
+                affected_version_range=affected_version_range,
+                package_versions=valid_versions,
+                ignorable_versions=IGNORE_TAGS,
+            )
+            affected_purls = [
+                PackageURL(type=pkg_type, namespace=pkg_namespace, name=pkg_name, version=version)
+                for version in aff_vers
+            ]
+
+            unaffected_purls = [
+                PackageURL(type=pkg_type, namespace=pkg_namespace, name=pkg_name, version=version)
+                for version in unaff_vers
+            ]
+
+            affected_packages: List[LegacyAffectedPackage] = nearest_patched_package(
+                vulnerable_packages=affected_purls, resolved_packages=unaffected_purls
+            )
+
+            for (
+                fixed_package,
+                affected_packages,
+            ) in get_affected_packages_by_patched_package(affected_packages).items():
+                yield Inference.from_advisory_data(
+                    advisory_data,
+                    confidence=100,  # We are getting all valid versions to get this inference
+                    affected_purls=affected_packages,
+                    fixed_purl=fixed_package,
+                )

vulnerabilities/importers/istio.py

@@ -6,13 +6,20 @@
 # See https://github.com/nexB/vulnerablecode for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
+import logging
 import re
+from datetime import datetime
 from pathlib import Path
+from typing import Iterable
+from typing import List
+from typing import Mapping
+from typing import Optional
 from typing import Set
 
 import pytz
 import saneyaml
 from dateutil import parser
+from django.db.models.query import QuerySet
 from packageurl import PackageURL
 from univers.version_constraint import VersionConstraint
 from univers.version_range import GitHubVersionRange
@@ -23,10 +30,22 @@
 from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
+from vulnerabilities.importer import UnMergeablePackageError
+from vulnerabilities.improver import Improver
+from vulnerabilities.improver import Inference
+from vulnerabilities.models import Advisory
+from vulnerabilities.package_managers import GitHubTagsAPI
+from vulnerabilities.package_managers import VersionAPI
+from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage
+from vulnerabilities.utils import get_affected_packages_by_patched_package
+from vulnerabilities.utils import nearest_patched_package
+from vulnerabilities.utils import resolve_version_range
 from vulnerabilities.utils import split_markdown_front_matter
 
 is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match
 
+logger = logging.getLogger(__name__)
+
 
 class IstioImporter(Importer):
     spdx_license_expression = "Apache-2.0"
@@ -136,3 +155,70 @@ def get_data_from_md(self, path):
         with open(path) as f:
             front_matter, _ = split_markdown_front_matter(f.read())
             return saneyaml.load(front_matter)
+
+
+class IstioImprover(Improver):
+    def __init__(self) -> None:
+        self.versions_fetcher_by_purl: Mapping[str, VersionAPI] = {}
+        self.vesions_by_purl = {}
+
+    @property
+    def interesting_advisories(self) -> QuerySet:
+        return Advisory.objects.filter(created_by=IstioImporter.qualified_name)
+
+    def get_package_versions(
+        self, package_url: PackageURL, until: Optional[datetime] = None
+    ) -> List[str]:
+        """
+        Return a list of `valid_versions` for the `package_url`
+        """
+        api_name = "istio/istio"
+        versions_fetcher = GitHubTagsAPI()
+        return versions_fetcher.get_until(package_name=api_name, until=until).valid_versions
+
+    def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]:
+        """
+        Yield Inferences for the given advisory data
+        """
+        if not advisory_data.affected_packages:
+            return
+        for affected_package in advisory_data.affected_packages:
+            purl = affected_package.package
+            affected_version_range = affected_package.affected_version_range
+            pkg_type = purl.type
+            pkg_namespace = purl.namespace
+            pkg_name = purl.name
+            if not self.vesions_by_purl.get("istio/istio"):
+                valid_versions = self.get_package_versions(
+                    package_url=purl, until=advisory_data.date_published
+                )
+                self.vesions_by_purl["istio/istio"] = valid_versions
+            valid_versions = self.vesions_by_purl["istio/istio"]
+            aff_vers, unaff_vers = resolve_version_range(
+                affected_version_range=affected_version_range,
+                package_versions=valid_versions,
+            )
+            affected_purls = [
+                PackageURL(type=pkg_type, namespace=pkg_namespace, name=pkg_name, version=version)
+                for version in aff_vers
+            ]
+
+            unaffected_purls = [
+                PackageURL(type=pkg_type, namespace=pkg_namespace, name=pkg_name, version=version)
+                for version in unaff_vers
+            ]
+
+            affected_packages: List[LegacyAffectedPackage] = nearest_patched_package(
+                vulnerable_packages=affected_purls, resolved_packages=unaffected_purls
+            )
+
+            for (
+                fixed_package,
+                affected_packages,
+            ) in get_affected_packages_by_patched_package(affected_packages).items():
+                yield Inference.from_advisory_data(
+                    advisory_data,
+                    confidence=100,  # We are getting all valid versions to get this inference
+                    affected_purls=affected_packages,
+                    fixed_purl=fixed_package,
+                )