Merge pull request #1621 from aboutcode-org/document-pipeline-design

Add documentation for new pipeline design

Author: keshav-space

Makefile

@@ -125,7 +125,7 @@ bump:
 
 docs:
 	rm -rf docs/_build/
-	@${ACTIVATE} sphinx-build docs/ docs/_build/
+	@${ACTIVATE} sphinx-build docs/source docs/_build/
 
 docker-images:
 	@echo "-> Build Docker services"

README.rst

@@ -38,7 +38,6 @@ we are trying to change this and evolve the status quo in a few other areas!
 
 Package URL themselves were designed first in ScanCode and VulnerableCode
 and are now a de-facto standard for vulnerability management and package references.
-
 See https://github.com/package-url/purl-spec
 
 The VulnerableCode project is a FOSS community resource to help improve the
@@ -47,34 +46,39 @@ security of the open source software ecosystem and its users at large.
 VulnerableCode consists of a database and the tools to collect, refine and keep
 the database current. 
 
-.. warning::
-    VulnerableCode is under active development and is not yet fully
-    usable.
 
-Read more about VulnerableCode https://vulnerablecode.readthedocs.org/
+.. pull-quote::
+   **Warning**
+
+   VulnerableCode is under active development and is not yet fully
+   usable.
 
-VulnerableCode is financially supported by NLnet, nexB, Google (through the
-GSoC) and the active contributions of several volunteers.
+
+Read more about VulnerableCode https://vulnerablecode.readthedocs.org/
 
 VulnerableCode tech stack is Python, Django, PostgreSQL, nginx and Docker and
 several libraries.
 
 
 Getting started
----------------
+===============
 
 Run with Docker
-^^^^^^^^^^^^^^^^
+---------------
+
+First install docker, then run
 
-First install docker, then run::
+.. code:: bash
 
     git clone https://github.com/nexB/vulnerablecode.git && cd vulnerablecode
     make envfile
     docker compose build
     docker compose up -d
     docker compose run vulnerablecode ./manage.py import --list
 
-Then run an importer for nginx advisories (which is small)::
+Then run an importer for nginx advisories (which is small)
+
+.. code:: bash
 
     docker compose exec vulnerablecode ./manage.py import vulnerabilities.importers.nginx.NginxImporter
     docker compose exec vulnerablecode ./manage.py improve --all
@@ -84,22 +88,29 @@ some data at http://localhost
 
 
 Populate VulnerableCode database
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+--------------------------------
 
 VulnerableCode data collection works in two steps: importing data from multiple
 sources and then refining and improving how package and software vulnerabilities
 are related.
 
-To run all importers and improvers use this::
+To run all importers and improvers use this
+
+.. code:: bash
 
    ./manage.py import --all
+
+.. code:: bash
+
    ./manage.py improve --all
 
 
 Local development installation
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+------------------------------
+
+On a Debian system, use this
 
-On a Debian system, use this::
+.. code:: bash
 
     sudo apt-get install  python3-venv python3-dev postgresql libpq-dev build-essential
     git clone https://github.com/nexB/vulnerablecode.git && cd vulnerablecode
@@ -112,22 +123,9 @@ On a Debian system, use this::
 
 At this point, the VulnerableCode app and API is up at http://127.0.0.1:8001/
 
-Interface
-^^^^^^^^^^
-
-
-VulnerableCode comes with a minimal web UI:
-
-.. image:: vulnerablecode-ui.png
-
-And a JSON API and its minimal web documentation:
-
-.. image:: vulnerablecode-json-api.png
-.. image:: vulnerablecode-api-doc.png
-
 
 License
-^^^^^^^^^^
+========
 
 Copyright (c) nexB Inc. and others. All rights reserved.
 
@@ -149,7 +147,7 @@ See https://aboutcode.org for more information about nexB OSS projects.
 
 
 Acknowledgements, Funding, Support and Sponsoring
---------------------------------------------------------
+=================================================
 
 This project is funded, supported and sponsored by:
 
@@ -281,5 +279,3 @@ Communications Networks, Content and Technology under grant agreement No 1010695
     :target: https://nlnet.nl/discovery/
     :height: 40
     :alt: NGI Discovery logo
-
-

docs/Makefile

@@ -5,6 +5,7 @@
 # from the environment for the first two.
 SPHINXOPTS    ?=
 SPHINXBUILD   ?= sphinx-build
+SPHINXAUTOBUILD = sphinx-autobuild
 SOURCEDIR     = source
 BUILDDIR      = build
 
@@ -14,6 +15,13 @@ help:
 
 .PHONY: help Makefile
 
+# Run the development server using sphinx-autobuild
+docs:
+	@echo
+	@echo "Starting up the docs server..."
+	@echo
+	$(SPHINXAUTOBUILD) --port 8000 --watch ${SOURCEDIR} $(SOURCEDIR) "$(BUILDDIR)/html" $(SPHINXOPTS) $(O)
+
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile

docs/source/README.gif

@@ -35,6 +35,7 @@
     "https://anongit.gentoo.org/git/data/glsa.git",  # Git only link
     "https://www.softwaretestinghelp.com/how-to-write-good-bug-report/",  # Cloudflare protection
     "https://www.openssl.org/news/vulnerabilities.xml",  # OpenSSL legacy advisory URL, not longer available
+    "https://example.org/api/non-existent-packages",
 ]
 
 # Add any Sphinx extension module names here, as strings. They can be