Merge remote-tracking branch 'skeleton/main' into main

Author: pombredanne

.github/workflows/pypi-release.yml

@@ -1,27 +1,83 @@
-name: Release library as a PyPI wheel and sdist on GH release creation
+name: Create library release archives, create a GH release and publish PyPI wheel and sdist on tag in main branch
+
+
+# This is executed automatically on a tag in the main branch
+
+# Summary of the steps:
+# - build wheels and sdist
+# - upload wheels and sdist to PyPI
+# - create gh-release and upload wheels and dists there
+# TODO: smoke test wheels and sdist
+# TODO: add changelog to release text body
+
+# WARNING: this is designed only for packages building as pure Python wheels
 
 on:
-  release:
-    types: [created]
+  workflow_dispatch:
+  push:
+    tags:
+      - "v*.*.*"
 
 jobs:
-  build-and-publish-to-pypi:
+  build-pypi-distribs:
     name: Build and publish library to PyPI
     runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@master
+      - name: Set up Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.9
+
+      - name: Install pypa/build
+        run: python -m pip install build --user
+
+      - name: Build a binary wheel and a source tarball
+        run: python -m build --sdist --wheel --outdir dist/
+
+      - name: Upload built archives
+        uses: actions/upload-artifact@v3
+        with:
+          name: pypi_archives
+          path: dist/*
+
+
+  create-gh-release:
+    name: Create GH release
+    needs:
+      - build-pypi-distribs
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Download built archives
+        uses: actions/download-artifact@v3
+        with:
+          name: pypi_archives
+          path: dist
+
+      - name: Create GH release
+        uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          files: dist/*
+
+
+  create-pypi-release:
+    name: Create PyPI release
+    needs:
+      - create-gh-release
+    runs-on: ubuntu-20.04
+
     steps:
-     - uses: actions/checkout@master
-     - name: Set up Python
-       uses: actions/setup-python@v1
-       with:
-         python-version: 3.9
-     - name: Install pypa/build
-       run: python -m pip install build --user
-     - name: Build a binary wheel and a source tarball
-       run: python -m build --sdist --wheel --outdir dist/
-        .
-     - name: Publish distribution  to PyPI
-       if: startsWith(github.ref, 'refs/tags')
-       uses: pypa/gh-action-pypi-publish@master
-       with:
-         password: ${{ secrets.PYPI_API_TOKEN }}
-        
+      - name: Download built archives
+        uses: actions/download-artifact@v3
+        with:
+          name: pypi_archives
+          path: dist
+
+      - name: Publish to PyPI
+        if: startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@master
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

configure

@@ -52,13 +52,20 @@ CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
 
 
+################################
+# Install with or without and index. With "--no-index" this is using only local wheels
+# This is an offline mode with no index and no network operations
+# NO_INDEX="--no-index "
+NO_INDEX=""
+
+
 ################################
 # Thirdparty package locations and index handling
-# Find packages from the local thirdparty directory or from thirdparty.aboutcode.org
-if [ -d "$CFG_ROOT_DIR/thirdparty" ]; then
-    PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty"
+# Find packages from the local thirdparty directory if present
+THIRDPARDIR=$CFG_ROOT_DIR/thirdparty
+if [[ "$(echo $THIRDPARDIR/*.whl)x" != "$THIRDPARDIR/*.whlx" ]]; then
+    PIP_EXTRA_ARGS="$NO_INDEX --find-links $THIRDPARDIR"
 fi
-PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html"
 
 
 ################################
@@ -183,6 +190,7 @@ while getopts :-: optchar; do
     esac
 done
 
+
 PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS"
 
 find_python

configure.bat

@@ -52,11 +52,10 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts"
 
 @rem ################################
 @rem # Thirdparty package locations and index handling
-@rem # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org
+@rem # Find packages from the local thirdparty directory
 if exist "%CFG_ROOT_DIR%\thirdparty" (
     set PIP_EXTRA_ARGS=--find-links "%CFG_ROOT_DIR%\thirdparty"
 )
-set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html"
 
 
 @rem ################################
@@ -69,7 +68,6 @@ if not defined CFG_QUIET (
 @rem ################################
 @rem # Main command line entry point
 set "CFG_REQUIREMENTS=%REQUIREMENTS%"
-set "NO_INDEX=--no-index"
 
 :again
 if not "%1" == "" (

docs/source/conf.py

@@ -28,16 +28,16 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-'sphinx.ext.intersphinx',
+    "sphinx.ext.intersphinx",
 ]
 
 # This points to aboutcode.readthedocs.io
 # In case of "undefined label" ERRORS check docs on intersphinx to troubleshoot
 # Link was created at commit - https://github.com/nexB/aboutcode/commit/faea9fcf3248f8f198844fe34d43833224ac4a83
 
 intersphinx_mapping = {
-    'aboutcode': ('https://aboutcode.readthedocs.io/en/latest/', None),
-    'scancode-workbench': ('https://scancode-workbench.readthedocs.io/en/develop/', None),
+    "aboutcode": ("https://aboutcode.readthedocs.io/en/latest/", None),
+    "scancode-workbench": ("https://scancode-workbench.readthedocs.io/en/develop/", None),
 }
 
 
@@ -62,7 +62,7 @@
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ["_static"]
 
-master_doc = 'index'
+master_doc = "index"
 
 html_context = {
     "display_github": True,
@@ -72,9 +72,7 @@
     "conf_py_path": "/docs/source/",  # path in the checkout to the docs root
 }
 
-html_css_files = [
-    '_static/theme_overrides.css'
-    ]
+html_css_files = ["_static/theme_overrides.css"]
 
 
 # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.

etc/scripts/fetch_thirdparty.py

@@ -110,7 +110,6 @@
     is_flag=True,
     help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.",
 )
-
 @click.help_option("-h", "--help")
 def fetch_thirdparty(
     requirements_files,

etc/scripts/utils_thirdparty.py

@@ -311,6 +311,7 @@ def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()):
 
     return fetched_sdist_filename
 
+
 ################################################################################
 #
 # Core models
@@ -1064,16 +1065,16 @@ def get_sdist_name_ver_ext(filename):
     if version.isalpha():
         return False
 
-    # non-pep 440 version    
+    # non-pep 440 version
     if "-" in version:
         return False
 
-    # single version    
+    # single version
     if version.isdigit() and len(version) == 1:
         return False
 
-    # r1 version    
-    if len(version) == 2 and version[0]=="r" and version[1].isdigit():
+    # r1 version
+    if len(version) == 2 and version[0] == "r" and version[1].isdigit():
         return False
 
     # dotless version (but calver is OK)
@@ -1588,6 +1589,7 @@ def tags(self):
             )
         )
 
+
 ################################################################################
 #
 # PyPI repo and link index for package wheels and sources
@@ -1629,7 +1631,9 @@ class PypiSimpleRepository:
     use_cached_index = attr.ib(
         type=bool,
         default=False,
-        metadata=dict(help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache."),
+        metadata=dict(
+            help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache."
+        ),
     )
 
     def _get_package_versions_map(self, name):
@@ -1674,6 +1678,7 @@ def get_package_version(self, name, version=None):
         """
         if not version:
             versions = list(self._get_package_versions_map(name).values())
+            # return the latest version
             return versions and versions[-1]
         else:
             return self._get_package_versions_map(name).get(version)
@@ -1724,7 +1729,9 @@ class LinksRepository:
     use_cached_index = attr.ib(
         type=bool,
         default=False,
-        metadata=dict(help="If True, use any existing on-disk cached index files. Otherwise, fetch and cache."),
+        metadata=dict(
+            help="If True, use any existing on-disk cached index files. Otherwise, fetch and cache."
+        ),
     )
 
     def __attrs_post_init__(self):
@@ -1790,6 +1797,7 @@ def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}, use_cached_index=False):
             _LINKS_REPO[url] = cls(url=url, use_cached_index=use_cached_index)
         return _LINKS_REPO[url]
 
+
 ################################################################################
 # Globals for remote repos to be lazily created and cached on first use for the
 # life of the session together with some convenience functions.
@@ -1803,6 +1811,7 @@ def get_local_packages(directory=THIRDPARTY_DIR):
     """
     return list(PypiPackage.packages_from_dir(directory=directory))
 
+
 ################################################################################
 #
 # Basic file and URL-based operations using a persistent file-based Cache
@@ -1953,6 +1962,7 @@ def fetch_and_save(
         fo.write(content)
     return content
 
+
 ################################################################################
 #
 # Functions to update or fetch ABOUT and license files
@@ -2051,7 +2061,9 @@ def get_other_dists(_package, _dist):
                 # if has key data we may look to improve later, but we can move on
                 if local_dist.has_key_metadata():
                     local_dist.save_about_and_notice_files(dest_dir=dest_dir)
-                    local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index)
+                    local_dist.fetch_license_files(
+                        dest_dir=dest_dir, use_cached_index=use_cached_index
+                    )
                     continue
 
             # lets try to fetch remotely
@@ -2089,7 +2101,9 @@ def get_other_dists(_package, _dist):
                 # if has key data we may look to improve later, but we can move on
                 if local_dist.has_key_metadata():
                     local_dist.save_about_and_notice_files(dest_dir=dest_dir)
-                    local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index)
+                    local_dist.fetch_license_files(
+                        dest_dir=dest_dir, use_cached_index=use_cached_index
+                    )
                     continue
 
             # try to get data from pkginfo (no license though)
@@ -2107,6 +2121,7 @@ def get_other_dists(_package, _dist):
                 lic_errs = "\n".join(lic_errs)
                 print(f"Failed to fetch some licenses:: {lic_errs}")
 
+
 ################################################################################
 #
 # Functions to build new Python wheels including native on multiple OSes
@@ -2210,6 +2225,7 @@ def download_wheels_with_pip(
     downloaded = existing ^ set(os.listdir(dest_dir))
     return sorted(downloaded), error
 
+
 ################################################################################
 #
 # Functions to check for problems