Merge branch 'develop' of github.com:nexB/aboutcode-toolkit into develop

chinyeungli · chinyeungli · commit edcac62492fa · 2023-07-14T18:11:23.000+08:00
diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml
@@ -13,10 +13,10 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml
@@ -24,9 +24,9 @@ jobs:
     runs-on: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v3
       - name: Set up Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: 3.9
 
@@ -78,6 +78,6 @@ jobs:
 
       - name: Publish to PyPI
         if: startsWith(github.ref, 'refs/tags')
-        uses: pypa/gh-action-pypi-publish@master
+        uses: pypa/gh-action-pypi-publish@release/v1
         with:
           password: ${{ secrets.PYPI_API_TOKEN }}
diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -5,6 +5,17 @@
 # Required
 version: 2
 
+# Build in latest ubuntu/python
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+# Build PDF & ePub
+formats:
+  - epub
+  - pdf
+
 # Where the Sphinx conf.py file is located
 sphinx:
    configuration: docs/source/conf.py
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,10 +1,12 @@
 ==============================
 Changelog
+
 2023-xx-xx
     Release x.x.x
 
     * The tool will now show which worksheet (if .xlsx input) is the tool working on
     * Error handling if defined worksheet does not exist
+    * Adopt 3.3.1 specification: introduce ``ignored_resources``
 
 
 2023-03-09
diff --git a/README.rst b/README.rst
@@ -21,7 +21,7 @@ In addition, this tool is able to generate attribution notices and
 identify redistributable source code used in your project to help you comply
 with open source licenses conditions.
 
-This version of the AboutCode Toolkit follows the ABOUT specification version 3.3.0 at:
+This version of the AboutCode Toolkit follows the ABOUT specification version 3.3.1 at:
 https://aboutcode-toolkit.readthedocs.io/en/latest/specification.html
 
 
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -7,14 +7,6 @@
 
 jobs:
 
-    - template: etc/ci/azure-posix.yml
-      parameters:
-          job_name: ubuntu18_cpython
-          image_name: ubuntu-18.04
-          python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11']
-          test_suites:
-              all: venv/bin/pytest -n 2 -vvs
-
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: ubuntu20_cpython
@@ -31,14 +23,6 @@ jobs:
           test_suites:
               all: venv/bin/pytest -n 2 -vvs
 
-    - template: etc/ci/azure-posix.yml
-      parameters:
-          job_name: macos1015_cpython_2
-          image_name: macos-10.15
-          python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11']
-          test_suites:
-              all: venv/bin/pytest -n 2 -vvs
-
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: macos11_cpython
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -28,9 +28,25 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-'sphinx.ext.intersphinx',
+    "sphinx.ext.intersphinx",
+    "sphinx_reredirects",
 ]
 
+
+# Redirects for olds pages
+# See https://documatt.gitlab.io/sphinx-reredirects/usage.html
+redirects = {}
+
+# This points to aboutcode.readthedocs.io
+# In case of "undefined label" ERRORS check docs on intersphinx to troubleshoot
+# Link was created at commit - https://github.com/nexB/aboutcode/commit/faea9fcf3248f8f198844fe34d43833224ac4a83
+
+intersphinx_mapping = {
+    "aboutcode": ("https://aboutcode.readthedocs.io/en/latest/", None),
+    "scancode-workbench": ("https://scancode-workbench.readthedocs.io/en/develop/", None),
+}
+
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
@@ -61,6 +77,32 @@
     "conf_py_path": "/docs/source/",  # path in the checkout to the docs root
     }
 
-html_css_files = [
-    '_static/theme_overrides.css'
-    ]
+html_css_files = ["_static/theme_overrides.css"]
+
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+html_show_sphinx = True
+
+# Define CSS and HTML abbreviations used in .rst files.  These are examples.
+# .. role:: is used to refer to styles defined in _static/theme_overrides.css and is used like this: :red:`text`
+rst_prolog = """
+.. |psf| replace:: Python Software Foundation
+
+.. # define a hard line break for HTML
+.. |br| raw:: html
+
+   <br />
+
+.. role:: red
+
+.. role:: img-title
+
+.. role:: img-title-para
+
+"""
+
+# -- Options for LaTeX output -------------------------------------------------
+
+latex_elements = {
+    'classoptions': ',openany,oneside'
+}
diff --git a/docs/source/general.rst b/docs/source/general.rst
@@ -85,6 +85,9 @@ it will copy and store next to the .ABOUT files.
     * - name
       - Component name
       - Mandatory
+    * - ignored_resources
+      - List of paths ignored from the ``about_resource``
+      - Optional
     * - version
       - Component version
       - Optional
diff --git a/docs/source/home.rst b/docs/source/home.rst
@@ -20,7 +20,7 @@ In addition, this tool is able to generate attribution notices and
 identify redistributable source code used in your project to help you comply
 with open source licenses conditions.
 
-This version of the AboutCode Toolkit follows the ABOUT specification version 3.3.0 at:
+This version of the AboutCode Toolkit follows the ABOUT specification version 3.3.1 at:
 https://aboutcode-toolkit.readthedocs.io/en/latest/specification.html
 
 
diff --git a/docs/source/specification.rst b/docs/source/specification.rst
@@ -1,7 +1,7 @@
 .. _specification:
 
 ===============================
-ABOUT File Specification v3.3.0
+ABOUT File Specification v3.3.1
 ===============================
 
 Purpose
@@ -218,11 +218,14 @@ in any case combination.
 Referencing the file or directory documented by an ABOUT file
 -------------------------------------------------------------
 
-An ABOUT file documents one file or directory. The mandatory "about_resource"
-field reference the documented file or directory. The value of the "about_resource"
-field is the name or path of the referenced file or directory.
+An ABOUT file documents one file or directory. The mandatory ``about_resource``
+field reference the documented file or directory. The value of the ``about_resource``
+field is the name or path of the referenced file or directory. There is also a
+``ignored_resources`` field which can be used to ignore a set of subpaths inside the
+directory which is being documented in the ABOUT file.
 
-A tool processing an ABOUT file must report an error if this field is missing.
+A tool processing an ABOUT file must report an error if the ``about_resource``
+field is missing.
 
 By convention, an ABOUT file is often stored in the same directory side-by-side
 to the file or directory that it documents, but this is not mandatory.
@@ -240,6 +243,14 @@ In this example, the ABOUT file documents a whole sub-directory:
 
                 about_resource: linux-kernel-2.6.23
 
+In this example, the ABOUT file documents a whole sub-directory, with some
+sub-paths under the directory ignored:
+
+        ..  code-block:: none
+
+                about_resource: linux-kernel-2.6.23
+                ignored_resources: linux-kernel-2.6.23/Documentation
+
 In this example, the ABOUT file documents the current directory, using a "." period to reference it:
 
         ..  code-block:: none
@@ -258,6 +269,9 @@ mandatory field are missing.
 Optional Information fields
 ---------------------------
 
+-   ignored_resources: A list of paths under the ``about_resource`` path, which are
+    not documented in the ABOUT file, and the information in the ABOUT file does not
+    apply to these subpaths.
 -   version: Component or package version. A component or package usually has a version,
     such as a revision number or hash from a version control system (for a snapshot checked
     out from VCS such as Subversion or Git). If not available, the version should be the date
diff --git a/setup.cfg b/setup.cfg
@@ -82,9 +82,10 @@ testing =
     isort
 
 docs =
-    Sphinx >= 3.3.1
-    sphinx-rtd-theme >= 0.5.0
-    doc8 >= 0.8.1
+    Sphinx>=5.0.2
+    sphinx-rtd-theme>=1.0.0
+    sphinx-reredirects >= 0.1.2
+    doc8>=0.11.2
 
 [options.entry_points]
 console_scripts =
diff --git a/src/attributecode/__init__.py b/src/attributecode/__init__.py
@@ -22,7 +22,7 @@
 
 __version__ = '8.0.0'
 
-__about_spec_version__ = '3.3.0'
+__about_spec_version__ = '3.3.1'
 
 __copyright__ = """
 Copyright (c) nexB Inc. All rights reserved. http://dejacode.org
diff --git a/src/attributecode/model.py b/src/attributecode/model.py
@@ -558,6 +558,22 @@ def _validate(self, *args, **kwargs):
         return errors
 
 
+class IgnoredResourcesField(PathField):
+    """
+    Special field for ignored_resources. self.ignored_paths contains a list of
+    path patterns (glob patterns) which are not part of the summarization provided
+    by the ABOUT file. 
+    """
+
+    def __init__(self, *args, ** kwargs):
+        super(AboutResourceField, self).__init__(*args, ** kwargs)
+        self.resolved_paths = []
+
+    def _validate(self, *args, **kwargs):
+        errors = super(AboutResourceField, self)._validate(*args, ** kwargs)
+        return errors
+
+
 class FileTextField(PathField):
     """
     A path field pointing to one or more text files such as license files.
@@ -764,6 +780,7 @@ def set_standard_fields(self):
         """
         self.fields = dict([
             ('about_resource', AboutResourceField(required=True)),
+            ('ignored_resources', AboutResourceField()),
             ('name', SingleLineField(required=True)),
             ('version', SingleLineField()),
 
diff --git a/tests/test_model.py b/tests/test_model.py
@@ -456,6 +456,17 @@ def test_About_with_existing_about_resource_has_no_error(self):
         # this means we have a location
         self.assertNotEqual([], result)
 
+    def test_About_loads_ignored_resources_field(self):
+        # fields in this file are not in the standard order
+        test_file = get_test_loc('test_model/parse/with_ignored_resources.ABOUT')
+        a = model.About(test_file)
+        #assert [] == a.errors
+
+        expected = ['about_resource', 'ignored_resources', 'name']
+        result = [f.name for f in a.all_fields() if f.present]
+        assert expected == result
+
+
     def test_About_has_errors_when_about_resource_is_missing(self):
         test_file = get_test_loc('test_gen/parser_tests/.ABOUT')
         a = model.About(test_file)
diff --git a/tests/testdata/test_model/parse/with_ignored_resources.ABOUT b/tests/testdata/test_model/parse/with_ignored_resources.ABOUT
@@ -0,0 +1,5 @@
+name: elasticsearch-sidecar
+about_resource: elasticsearch-sidecar
+ignored_resources:
+    - elasticsearch-sidecar/plugins/
+    - elasticsearch-sidecar/logs/
