Improve release creation speed

pombredanne · pombredanne · commit 7b8215dee3de · 2021-08-05T17:58:24.000+02:00
- Allow to work from a checkout of wheels
- Add new --continue option to restart archive creation where left

Signed-off-by: Philippe Ombredanne &lt;pombredanne@nexb.com&gt;
diff --git a/configure b/configure
@@ -67,12 +67,17 @@ CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
 
 ################################
 # scancode-specific: Thirdparty package locations and index handling
-# Do we have thirdparty/files? use the mit.LICENSE as a proxy
-if [[ -f "$CFG_ROOT_DIR/thirdparty/mit.LICENSE" ]]; then
-    LINKS=$CFG_ROOT_DIR/thirdparty
+if [[ "$PYPI_LINKS" == "" ]]; then
+    # Do we have thirdparty/files? use the mit.LICENSE as a proxy
+    if [[ -f "$CFG_ROOT_DIR/thirdparty/mit.LICENSE" ]]; then
+        LINKS=$CFG_ROOT_DIR/thirdparty
+    else
+        LINKS=https://thirdparty.aboutcode.org/pypi
+    fi
 else
-    LINKS=https://thirdparty.aboutcode.org/pypi
+    LINKS=$PYPI_LINKS
 fi
+
 PIP_EXTRA_ARGS="--no-index --find-links $LINKS"
 ################################
 
diff --git a/etc/release/fetch_requirements.py b/etc/release/fetch_requirements.py
@@ -8,10 +8,11 @@
 # See https://github.com/nexB/scancode-toolkit for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
+import itertools
+
 import click
 
 import utils_thirdparty
-import itertools
 
 
 @click.command()
@@ -63,6 +64,15 @@
     is_flag=True,
     help='Fetch only the corresponding source distributions.',
 )
+@click.option('-u', '--remote-links-url',
+    type=str,
+    metavar='URL',
+    default=utils_thirdparty.REMOTE_LINKS_URL,
+    show_default=True,
+    help='URL to a PyPI-like links web site. '
+         'Or local path to a directory with wheels.',
+)
+
 @click.help_option('-h', '--help')
 def fetch_requirements(
     requirements_file,
@@ -73,6 +83,7 @@ def fetch_requirements(
     with_about,
     allow_unpinned,
     only_sources,
+    remote_links_url=utils_thirdparty.REMOTE_LINKS_URL,
 ):
     """
     Fetch and save to THIRDPARTY_DIR all the required wheels for pinned
@@ -82,7 +93,9 @@ def fetch_requirements(
     Also fetch the corresponding .ABOUT, .LICENSE and .NOTICE files together
     with a virtualenv.pyz app.
 
-    Use exclusively our remote repository (and not PyPI).
+    Use exclusively wheel not from PyPI but rather found in the PyPI-like link
+    repo ``remote_links_url`` if this is a URL. Treat this ``remote_links_url``
+    as a local directory path to a wheels directory if this is not a a URL.
     """
 
     # fetch wheels
@@ -93,23 +106,28 @@ def fetch_requirements(
     if not only_sources:
         envs = itertools.product(python_versions, operating_systems)
         envs = (utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in envs)
+
         for env, reqf in itertools.product(envs, requirements_files):
+            
             for package, error in utils_thirdparty.fetch_wheels(
                 environment=env,
                 requirements_file=reqf,
                 allow_unpinned=allow_unpinned,
                 dest_dir=thirdparty_dir,
+                remote_links_url=remote_links_url,
             ):
                 if error:
                     print('Failed to fetch wheel:', package, ':', error)
 
     # optionally fetch sources
     if with_sources or only_sources:
+
         for reqf in requirements_files:
             for package, error in utils_thirdparty.fetch_sources(
                 requirements_file=reqf,
                 allow_unpinned=allow_unpinned,
                 dest_dir=thirdparty_dir,
+                remote_links_url=remote_links_url,
             ):
                 if error:
                     print('Failed to fetch source:', package, ':', error)
diff --git a/etc/release/scancode-create-release.sh b/etc/release/scancode-create-release.sh
@@ -8,6 +8,11 @@
 # Create, test and publish release archives, wheels and sdists.
 # Use the --test to also run basic smoke tests of the built archives
 #
+# To use a local checkout of https://github.com/nexB/thirdparty-packages/ rather
+# than https://thirdparty.aboutcode.org/ set the variable PYPI_LINKS to point
+# to your thirdparty-packages/pypi local directory (this speeds up the release
+# creation and allow to work mostly offline.
+#
 ################################################################################
 
 # Supported current app Python version and OS
@@ -202,51 +207,36 @@ function clean_build {
     clean_egg_info
 }
 
-backup_previous_release
-clean_build
-mkdir release
-
-echo "## RELEASE: Setup environment"
-
-echo "## RELEASE: Clean and configure, then regen license index"
-./configure --clean
-./configure 
-source bin/activate
-scancode --reindex-licenses
-
-
-
-echo "## RELEASE: Install release requirements"
-# We do not need a full env for releasing
-bin/pip install $QUIET -r etc/release/requirements.txt
-
-
 ################################
 # PyPI wheels and sdist: these are not Python version- or OS-dependent
 ################################
-echo " "
-echo "## RELEASE: Building a wheel and a source distribution"
-clean_egg_info
-bin/python setup.py $QUIET sdist bdist_wheel
+function build_wheels {
+    # Build scancode wheels and dist for PyPI
+    # Arguments:
 
-mv dist release/pypi
+    echo " "
+    echo "## RELEASE: Building a wheel and a source distribution"
+    clean_egg_info
+    bin/python setup.py $QUIET sdist bdist_wheel
 
+    mv dist release/pypi
 
-echo " "
-echo "## RELEASE: Building a mini wheel and a source distribution"
-clean_egg_info
-mv setup.cfg setup-full.cfg
-cp setup-mini.cfg setup.cfg
-rm -rf build
-bin/python setup.py $QUIET sdist bdist_wheel
-mv setup-full.cfg setup.cfg
+    echo " "
+    echo "## RELEASE: Building a mini wheel and a source distribution"
+    clean_egg_info
+    mv setup.cfg setup-full.cfg
+    cp setup-mini.cfg setup.cfg
+    rm -rf build
+    bin/python setup.py $QUIET sdist bdist_wheel
+    mv setup-full.cfg setup.cfg
 
-cp dist/* release/pypi/
+    cp dist/* release/pypi/
 
-clean_egg_info
-echo "## RELEASE: full and mini, wheel and source distribution(s) built and ready for PyPI upload"
-find release -ls
+    clean_egg_info
+    echo "## RELEASE: full and mini, wheel and source distribution(s) built and ready for PyPI upload"
+    find release -ls
 
+}
 
 ################################
 # Build OSes and Pythons-specific release archives
@@ -268,34 +258,44 @@ function build_app_archive {
     echo " "
     echo "## RELEASE: Building archive for Python $python_app_version on operating system: $operating_system"
 
-    clean_build
-    mkdir -p thirdparty
-
     if [ "$operating_system" == "windows" ]; then
         # create a zip only on Windows
         formats=zip
-        echo -n "py -$python_app_dot_version">PYTHON_EXECUTABLE
+        formats_ext=.zip
     else
         formats=xztar
-        echo -n "python$python_app_dot_version">PYTHON_EXECUTABLE
+        formats_ext=.tar.xz
     fi
 
-    # 1. Collect thirdparty deps only for the subset for this Python/operating_system
-    bin/python etc/release/fetch_requirements.py \
-        --requirements-file=requirements.txt \
-        --thirdparty-dir=thirdparty \
-        --python-version=$python_app_version \
-        --operating-system=$operating_system \
-        --with-about
+    if [ ! -f release/archives/scancode-toolkit-*_py$python_app_version-$operating_system$formats_ext ]; then
+        clean_build
+        mkdir -p thirdparty
+
+        if [ "$operating_system" == "windows" ]; then
+            echo -n "py -$python_app_dot_version">PYTHON_EXECUTABLE
+        else
+            echo -n "python$python_app_dot_version">PYTHON_EXECUTABLE
+        fi
+
+        # 1. Collect thirdparty deps only for the subset for this Python/operating_system
+        bin/python etc/release/fetch_requirements.py \
+            --requirements-file=requirements.txt \
+            --thirdparty-dir=thirdparty \
+            --python-version=$python_app_version \
+            --operating-system=$operating_system \
+            --with-about \
+            --remote-links-url=$PYPI_LINKS
+
+        # 2. Create tarball or zip.
+        # For now as a shortcut we use the Python setup.py sdist to create a tarball.
+        # This is hackish and we should instead use our own archiving code that
+        # would take a distutils manifest-like input
+        bin/python setup.py $QUIET sdist --formats=$formats 
+        bin/python etc/release/scancode_rename_archives.py dist/ _py$python_app_version-$operating_system
+        mkdir -p release/archives
+        mv dist/* release/archives/
+    fi
 
-    # 2. Create tarball or zip.
-    # For now as a shortcut we use the Python setup.py sdist to create a tarball.
-    # This is hackish and we should instead use our own archiving code that
-    # would take a distutils manifest-like input
-    bin/python setup.py $QUIET sdist --formats=$formats 
-    bin/python etc/release/scancode_rename_archives.py dist/ _py$python_app_version-$operating_system
-    mkdir -p release/archives
-    mv dist/* release/archives/
 }
 
 
@@ -314,7 +314,8 @@ function build_source_archive {
         --requirements-file=requirements.txt \
         --thirdparty-dir=thirdparty \
         --with-about \
-        --only-sources
+        --only-sources \
+        --remote-links-url=$PYPI_LINKS
 
     # 2. Create tarball
     # For now as a shortcut we use the Python setup.py sdist to create a tarball.
@@ -328,6 +329,31 @@ function build_source_archive {
 }
 
 
+if [ "$CLI_ARGS" != "--continue" ]; then
+    echo "############# Reset release creation #############################"
+    backup_previous_release
+    clean_build
+    mkdir release
+
+    echo "## RELEASE: Setup environment"
+
+    echo "## RELEASE: Clean and configure, then regen license index"
+    ./configure --clean
+    PYPI_LINKS=$PYPI_LINKS ./configure --local 
+    source bin/activate
+    scancode --reindex-licenses
+
+    echo "## RELEASE: Install release requirements"
+    # We do not need a full env for releasing
+    bin/pip install $QUIET -r etc/release/requirements.txt
+else
+    echo "############# Continuing previous release creation #############################"
+fi
+
+
+# wheels
+build_wheels
+
 # build the app combos on the current App Python
 for operating_system in $OPERATING_SYSTEMS
     do
@@ -366,6 +392,5 @@ echo "###  RELEASE is ready for publishing  ###"
 # ping on chat and twitter
 # send email
 
-
 set +e
 set +x
diff --git a/etc/release/utils_thirdparty.py b/etc/release/utils_thirdparty.py
