Merge remote-tracking branch 'origin/main' into fix-ver-range

Author: pombredanne

.github/workflows/docs.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.8]
+        python-version: [3.9]
 
     steps:
       - name: Checkout code

.github/workflows/main.yml

@@ -29,7 +29,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
 
     steps:
       - name: Checkout code

.github/workflows/pypi-release.yml

@@ -37,7 +37,7 @@ jobs:
         run: python -m build --sdist --wheel --outdir dist/
 
       - name: Upload built archives
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: pypi_archives
           path: dist/*
@@ -51,7 +51,7 @@ jobs:
 
     steps:
       - name: Download built archives
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: pypi_archives
           path: dist
@@ -71,7 +71,7 @@ jobs:
 
     steps:
       - name: Download built archives
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: pypi_archives
           path: dist

.gitignore

@@ -46,8 +46,14 @@ coverage.xml
 *.log
 local_settings.py
 
-# Sphinx documentation
-docs/_build/
+# Sphinx
+docs/_build
+docs/bin
+docs/build
+docs/include
+docs/Lib
+doc/pyvenv.cfg
+pyvenv.cfg
 
 # PyBuilder
 target/
@@ -103,3 +109,13 @@ Pipfile
 *.bak
 /.cache/
 /tmp/
+
+# pyenv
+/.python-version
+/man/
+/.pytest_cache/
+lib64
+tcl
+
+# Ignore Jupyter Notebook related temp files
+.ipynb_checkpoints/

.readthedocs.yml

@@ -5,12 +5,25 @@
 # Required
 version: 2
 
+# Build in latest ubuntu/python
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+# Build PDF & ePub
+formats:
+  - epub
+  - pdf
+
 # Where the Sphinx conf.py file is located
 sphinx:
    configuration: docs/source/conf.py
 
-# Setting the doc build requirements
+# Setting the python version and doc build requirements
 python:
-  version: "3.7"
   install:
-    - requirements: docs/requirements.txt
+    - method: pip
+      path: .
+      extra_requirements:
+        - dev

CHANGELOG.rst

@@ -1,6 +1,172 @@
 Release notes
 =============
 
+Version v34.0.1
+-------------------
+
+- Add Pipeline to flag ghost packages (#1533)
+- Add logging configuration (#1533)
+- Drop support for python 3.8 (#1533)
+- Drop using docker-compose and use the built-in "docker compose" instead
+- Upgrade core dependencies including Django and Rest Framework
+- Fix typo in KEV improver (#1594)
+
+
+Version v34.0.0
+-------------------
+
+- Improve API performance.
+- Add severity range score in API.
+- Refactor GitlabDataSource to work with browser extension
+
+
+Version v34.0.0rc5
+-------------------
+
+- Add safetydb importer.
+- Add missing width setting for the table in the vulnerability details UI.
+- Add KEV support.
+- Add UI template for API.
+- Use VersionRange.normalize to compare advisory.
+- Use integer column to display score.
+- Add support for CVSSv4 & SSVC and import the data using vulnrichment.
+- Add support for reference_type in the API.
+- Add API improvements for the package endpoint.
+
+
+Version v34.0.0rc4
+-------------------
+
+- Drop migration for removing duplicated changelogs.
+
+
+Version v34.0.0rc3
+-------------------
+
+- Add resource URL to the vulnerability and package details view in the API serializers (#1423)
+- Add support for all osv ecosystems (#926)
+- Add RubyImporter to git_importer test_git_importer_clone (#799)
+- Remove duplicated changelogs (#1400)
+- Fix Encoding Type in Fireeye Importer (#1404)
+- Add license_url for GitHub Importer (#1392)
+- Add support for CVSS vectors display (#1312)
+
+
+Version v34.0.0rc2
+-------------------
+
+- We updated package-url models, WARNING: in next major version of 
+  vulnerablecode i.e v35.0.0 qualifiers will be of type ``string`` and not ``dict``.
+- We added changelog and dates on packages and vulnerabilities.
+- We fixed table borders in Vulnerability details UI #1356 (#1358)
+- We added robots.txt in views.
+- We fixed import runner's process_inferences (#1360)
+- We fixed debian OVAL importer (#1361)
+- We added graph model diagrams #977(#1350)
+- We added endpoint for purl lookup (#1359)
+- We fixed swagger API docs generation (#1366)
+- Fix issues https://github.com/nexB/vulnerablecode/issues/1385, https://github.com/nexB/vulnerablecode/issues/1387
+
+
+Version v34.0.0rc1
+-------------------
+
+- We updated package-url models, WARNING: in next major version of 
+  vulnerablecode i.e v35.0.0 qualifiers will be of type ``string`` and not ``dict``.
+- We added changelog and dates on packages and vulnerabilities.
+- We fixed table borders in Vulnerability details UI #1356 (#1358)
+- We added robots.txt in views.
+- We fixed import runner's process_inferences (#1360)
+- We fixed debian OVAL importer (#1361)
+- We added graph model diagrams #977(#1350)
+- We added endpoint for purl lookup (#1359)
+- We fixed swagger API docs generation (#1366)
+
+
+Version v33.6.5
+-------------------
+
+- We added /var/www/html as volume in nginx Docker compose (#1373).
+
+
+Version v33.6.4
+-------------------
+
+- We added /var/www/html as volume in Docker compose (#1371).
+
+
+Version v33.6.3
+----------------
+
+- We updated RTD build configuration.
+- We added importer for OSS-Fuzz.
+- We removed vulnerabilities with empty aliases.
+- We fixed search encoding issue https://github.com/nexB/vulnerablecode/issues/1336.
+- We added middleware to ban "bytedance" user-agent.
+
+
+Version v33.6.2
+----------------
+
+- We added note about CSRF_TRUSTED_ORIGINS.
+- We added proper acknowledgements for NGI projects.
+- We added throttling for anonymous users.
+
+Version v33.6.1
+----------------
+
+- We added pagination to valid versions improver.
+
+
+Version v33.6.0
+----------------
+
+- We added support to write packages and vulnerabilities at the time of import.
+
+
+Version v33.5.0
+----------------
+
+- We fixed a text-overflow issue in the Essentials tab of the Vulnerability details template.
+- We added clickable links to the Essentials tab of the Vulnerability details template that enable
+  the user to navigate to the Fixed by packages tab and the Affected packages tab.
+- We fixed severity range issue for handling unknown scores.
+
+Version v33.4.0
+----------------
+
+- We added importer specific improvers and removed default improver
+  additionally improve recent advisories first.
+
+
+Version v33.3.0
+----------------
+
+- We filtered out the weakness that are not presented in the
+  cwe2.database before passing them into the vulnerability details view.
+
+
+Version v33.2.0
+-----------------
+
+- We fixed NVD importer to import the latest data by adding weakness
+  in unique content ID for advisories.
+
+
+Version v33.1.0
+-----------------
+
+- We have paginated the default improver and added keyboard interrupt support for import and improve processes.
+- We bumped PyYaml to 6.0.1 and saneyaml to 0.6.0 and dropped docker-compose.
+
+
+Version v33.0.0
+-----------------
+
+- We have dropped ``unresolved_vulnerabilities`` from /api/package endpoint API response.
+- We have added missing quotes for href values in template.
+- We have fixed merge functionality of AffectedPackage.
+
 
 Version v32.0.1
 -----------------
@@ -20,9 +186,9 @@ Version v32.0.0rc4
 -------------------
 
 - We added loading of env for GitHub datasource in vulntotal.
-- We fixed import process in github importer in vulnerablecode reported here 
+- We fixed import process in github importer in vulnerablecode reported here
   https://github.com/nexB/vulnerablecode/issues/1142.
-- We added an improver to get all package versions 
+- We added an improver to get all package versions
   of all ecosystems for a range of affected packages.
 - We added documentation for configuring throttling rate for API endpoints.
 - We fixed kbmsr2019 importer.

Makefile

@@ -129,9 +129,9 @@ docs:
 
 docker-images:
 	@echo "-> Build Docker services"
-	docker-compose build
+	docker compose build
 	@echo "-> Pull service images"
-	docker-compose pull
+	docker compose pull
 	@echo "-> Save the service images to a compressed tar archive in the dist/ directory"
 	@mkdir -p dist/
 	@docker save postgres vulnerablecode_vulnerablecode nginx | gzip > dist/vulnerablecode-images-`git describe --tags`.tar.gz

README.rst

@@ -66,18 +66,18 @@ Getting started
 Run with Docker
 ^^^^^^^^^^^^^^^^
 
-First install docker and docker-compose, then run::
+First install docker, then run::
 
     git clone https://github.com/nexB/vulnerablecode.git && cd vulnerablecode
     make envfile
-    docker-compose build
-    docker-compose up -d
-    docker-compose run vulnerablecode ./manage.py import --list
+    docker compose build
+    docker compose up -d
+    docker compose run vulnerablecode ./manage.py import --list
 
 Then run an importer for nginx advisories (which is small)::
 
-    docker-compose exec vulnerablecode ./manage.py import vulnerabilities.importers.nginx.NginxImporter
-    docker-compose exec vulnerablecode ./manage.py improve --all
+    docker compose exec vulnerablecode ./manage.py import vulnerabilities.importers.nginx.NginxImporter
+    docker compose exec vulnerablecode ./manage.py improve --all
 
 At this point, the VulnerableCode app and API should be up and running with
 some data at http://localhost
@@ -105,6 +105,7 @@ On a Debian system, use this::
     git clone https://github.com/nexB/vulnerablecode.git && cd vulnerablecode
     make dev envfile postgres
     make test
+    source venv/bin/activate
     ./manage.py import vulnerabilities.importers.nginx.NginxImporter
     ./manage.py improve --all
     make run
@@ -145,3 +146,20 @@ See https://creativecommons.org/licenses/by-sa/4.0/legalcode for the license tex
 See https://github.com/nexB/vulnerablecode for support or download. 
 
 See https://aboutcode.org for more information about nexB OSS projects.
+
+Acknowledgements
+^^^^^^^^^^^^^^^^
+
+This project was funded through the NGI0 PET Fund, a fund established by
+NLnet with financial support from the European Commission's Next Generation
+Internet programme, under the aegis of DG Communications Networks, Content
+and Technology under grant agreement No 825310.
+
+https://nlnet.nl/project/VulnerableCode/
+
+This project was funded through the NGI0 Discovery Fund, a fund established
+by NLnet with financial support from the European Commission's Next Generation
+Internet programme, under the aegis of DG Communications Networks, Content
+and Technology under grant agreement No 825322.
+
+https://nlnet.nl/project/vulnerabilitydatabase/

SOURCES.rst

@@ -13,7 +13,7 @@
 +----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+
 |ruby            | https://github.com/rubysec/ruby-advisory-db.git                                                      |ruby gems                                           |
 +----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+
-|ubuntu          | https://people.canonical.com/~ubuntu-security/oval/                                                  |ubuntu packages                                     |
+|ubuntu          |                                                                                                      |ubuntu packages                                     |
 +----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+
 |retiredotnet    | https://github.com/RetireNet/Packages.git                                                            |.NET packages                                       |
 +----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+

aboutcode/hashid/README.rst

@@ -0,0 +1,15 @@
+aboutcode.hashid
+==================
+
+This is a library of utilities to compute ids and file paths for AboutCode  using VCID and PURLs.
+
+License
+-------
+
+Copyright (c) nexB Inc. and others. All rights reserved.
+
+SPDX-License-Identifier: Apache-2.0
+
+See https://github.com/aboutcode-org/vulnerablecode for support or download.
+
+See https://aboutcode.org for more information about AboutCode OSS projects.