Merge pull request #62 from nexB/better-tpp-scripts

Better tpp scripts

Author: pombredanne

README.rst

@@ -1,12 +1,13 @@
 A Simple Python Project Skeleton
 ================================
-This repo attempts to standardize our python repositories using modern python
-packaging and configuration techniques. Using this `blog post`_ as inspiration, this
-repository will serve as the base for all new python projects and will be adopted to all
-our existing ones as well.
+This repo attempts to standardize the structure of the Python-based project's
+repositories using modern Python packaging and configuration techniques. 
+Using this `blog post`_ as inspiration, this repository serves as the base for
+all new Python projects and is mergeable in existing repositories as well.
 
 .. _blog post: https://blog.jaraco.com/a-project-skeleton-for-python-projects/
 
+
 Usage
 =====
 
@@ -36,9 +37,16 @@ This is also the workflow to use when updating the skeleton files in any given r
 
 More usage instructions can be found in ``docs/skeleton-usage.rst``.
 
+
 Release Notes
 =============
 
+- 2022-03-04:
+    - Synchronize configure and configure.bat scripts for sanity
+    - Update CI operating system support with latest Azure OS images
+    - Streamline utility scripts in etc/scripts/ to create, fetch and manage third-party dependencies
+      There are now fewer scripts. See etc/scripts/README.rst for details
+
 - 2021-09-03:
     - ``configure`` now requires pinned dependencies via the use of ``requirements.txt`` and ``requirements-dev.txt``
     - ``configure`` can now accept multiple options at once

azure-pipelines.yml

@@ -31,6 +31,14 @@ jobs:
           test_suites:
               all: venv/bin/pytest -n 2 -vvs
 
+    - template: etc/ci/azure-posix.yml
+      parameters:
+          job_name: macos11_cpython
+          image_name: macos-11
+          python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10']
+          test_suites:
+              all: venv/bin/pytest -n 2 -vvs
+
     - template: etc/ci/azure-win.yml
       parameters:
           job_name: win2019_cpython

configure

@@ -16,6 +16,8 @@ set -e
 # Source this script for initial configuration
 # Use configure --help for details
 #
+# NOTE: please keep in sync with Windows script configure.bat
+#
 # This script will search for a virtualenv.pyz app in etc/thirdparty/virtualenv.pyz
 # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default
 ################################
@@ -32,10 +34,8 @@ DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constrai
 # where we create a virtualenv
 VIRTUALENV_DIR=venv
 
-# Cleanable files and directories with the --clean option
-CLEANABLE="
-    build
-	venv"
+# Cleanable files and directories to delete with the --clean option
+CLEANABLE="build venv"
 
 # extra  arguments passed to pip
 PIP_EXTRA_ARGS=" "
@@ -50,11 +50,14 @@ VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz
 CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
 
+
+################################
+# Thirdparty package locations and index handling
 # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org
 if [ -f "$CFG_ROOT_DIR/thirdparty" ]; then
-    PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty "
+    PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty"
 fi
-PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi"
+PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html"
 
 
 ################################
@@ -65,56 +68,50 @@ fi
 
 
 ################################
-# find a proper Python to run
-# Use environment variables or a file if available.
-# Otherwise the latest Python by default.
-if [[ "$PYTHON_EXECUTABLE" == "" ]]; then
-    # check for a file named PYTHON_EXECUTABLE
-    if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then
-        PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE")
-    else
-        PYTHON_EXECUTABLE=python3
-    fi
-fi
+# Main command line entry point
+main() {
+    CFG_REQUIREMENTS=$REQUIREMENTS
+    NO_INDEX="--no-index"
+    
+    # We are using getopts to parse option arguments that start with "-"
+    while getopts :-: optchar; do
+        case "${optchar}" in
+            -)
+                case "${OPTARG}" in
+                    help  ) cli_help;;
+                    clean ) find_python && clean;;
+                    dev   ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";;
+                    init  ) NO_INDEX="";;
+                esac;;
+        esac
+    done
 
+    PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX"
 
-################################
-cli_help() {
-    echo An initial configuration script
-    echo "  usage: ./configure [options]"
-    echo
-    echo The default is to configure for regular use. Use --dev for development.
-    echo Use the --init option if starting a new project and the project
-    echo dependencies are not available on thirdparty.aboutcode.org/pypi/
-    echo and requirements.txt and/or requirements-dev.txt has not been generated.
-    echo
-    echo The options are:
-    echo " --clean: clean built and installed files and exit."
-    echo " --dev:   configure the environment for development."
-    echo " --init:  pull dependencies from PyPI. Used when first setting up a project."
-    echo " --help:  display this help message and exit."
-    echo
-    echo By default, the python interpreter version found in the path is used.
-    echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to
-    echo configure another Python executable interpreter to use. If this is not
-    echo set, a file named PYTHON_EXECUTABLE containing a single line with the
-    echo path of the Python executable to use will be checked last.
-    set +e
-    exit
+    find_python
+    create_virtualenv "$VIRTUALENV_DIR"
+    install_packages "$CFG_REQUIREMENTS"
+    . "$CFG_BIN_DIR/activate"
 }
 
 
-clean() {
-    # Remove cleanable file and directories and files from the root dir.
-    echo "* Cleaning ..."
-    for cln in $CLEANABLE;
-        do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}";
-    done
-    set +e
-    exit
+################################
+# Find a proper Python to run
+# Use environment variables or a file if available.
+# Otherwise the latest Python by default.
+find_python() {
+    if [[ "$PYTHON_EXECUTABLE" == "" ]]; then
+        # check for a file named PYTHON_EXECUTABLE
+        if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then
+            PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE")
+        else
+            PYTHON_EXECUTABLE=python3
+        fi
+    fi
 }
 
 
+################################
 create_virtualenv() {
     # create a virtualenv for Python
     # Note: we do not use the bundled Python 3 "venv" because its behavior and
@@ -145,6 +142,7 @@ create_virtualenv() {
 }
 
 
+################################
 install_packages() {
     # install requirements in virtualenv
     # note: --no-build-isolation means that pip/wheel/setuptools will not
@@ -162,28 +160,43 @@ install_packages() {
 
 
 ################################
-# Main command line entry point
-CFG_DEV_MODE=0
-CFG_REQUIREMENTS=$REQUIREMENTS
-NO_INDEX="--no-index"
-
-# We are using getopts to parse option arguments that start with "-"
-while getopts :-: optchar; do
-    case "${optchar}" in
-        -)
-            case "${OPTARG}" in
-                help  ) cli_help;;
-                clean ) clean;;
-                dev   ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;;
-                init  ) NO_INDEX="";;
-            esac;;
-    esac
-done
-
-PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX"
-
-create_virtualenv "$VIRTUALENV_DIR"
-install_packages "$CFG_REQUIREMENTS"
-. "$CFG_BIN_DIR/activate"
+cli_help() {
+    echo An initial configuration script
+    echo "  usage: ./configure [options]"
+    echo
+    echo The default is to configure for regular use. Use --dev for development.
+    echo Use the --init option if starting a new project and the project
+    echo dependencies are not available on thirdparty.aboutcode.org/pypi/
+    echo and requirements.txt and/or requirements-dev.txt has not been generated.
+    echo
+    echo The options are:
+    echo " --clean: clean built and installed files and exit."
+    echo " --dev:   configure the environment for development."
+    echo " --init:  pull dependencies from PyPI. Used when first setting up a project."
+    echo " --help:  display this help message and exit."
+    echo
+    echo By default, the python interpreter version found in the path is used.
+    echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to
+    echo configure another Python executable interpreter to use. If this is not
+    echo set, a file named PYTHON_EXECUTABLE containing a single line with the
+    echo path of the Python executable to use will be checked last.
+    set +e
+    exit
+}
+
+
+################################
+clean() {
+    # Remove cleanable file and directories and files from the root dir.
+    echo "* Cleaning ..."
+    for cln in $CLEANABLE;
+        do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}";
+    done
+    set +e
+    exit
+}
+
+
+main
 
 set +e

configure.bat

@@ -14,6 +14,8 @@
 @rem # Source this script for initial configuration
 @rem # Use configure --help for details
 
+@rem # NOTE: please keep in sync with POSIX script configure
+
 @rem # This script will search for a virtualenv.pyz app in etc\thirdparty\virtualenv.pyz
 @rem # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default
 @rem ################################
@@ -49,10 +51,11 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts"
 
 @rem ################################
 @rem # Thirdparty package locations and index handling
+@rem # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org
 if exist "%CFG_ROOT_DIR%\thirdparty" (
     set PIP_EXTRA_ARGS=--find-links "%CFG_ROOT_DIR%\thirdparty"
 )
-set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi" & %INDEX_ARG%
+set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html"
 
 
 @rem ################################
@@ -64,7 +67,6 @@ if not defined CFG_QUIET (
 
 @rem ################################
 @rem # Main command line entry point
-set CFG_DEV_MODE=0
 set "CFG_REQUIREMENTS=%REQUIREMENTS%"
 set "NO_INDEX=--no-index"
 
@@ -74,7 +76,6 @@ if not "%1" == "" (
     if "%1" EQU "--clean"  (goto clean)
     if "%1" EQU "--dev"    (
         set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%"
-        set CFG_DEV_MODE=1
     )
     if "%1" EQU "--init"   (
         set "NO_INDEX= "
@@ -87,7 +88,7 @@ set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% %NO_INDEX%"
 
 
 @rem ################################
-@rem # find a proper Python to run
+@rem # Find a proper Python to run
 @rem # Use environment variables or a file if available.
 @rem # Otherwise the latest Python by default.
 if not defined PYTHON_EXECUTABLE (
@@ -99,6 +100,8 @@ if not defined PYTHON_EXECUTABLE (
     )
 )
 
+
+@rem ################################
 :create_virtualenv
 @rem # create a virtualenv for Python
 @rem # Note: we do not use the bundled Python 3 "venv" because its behavior and
@@ -143,6 +146,7 @@ if %ERRORLEVEL% neq 0 (
 )
 
 
+@rem ################################
 :install_packages
 @rem # install requirements in virtualenv
 @rem # note: --no-build-isolation means that pip/wheel/setuptools will not
@@ -157,6 +161,9 @@ if %ERRORLEVEL% neq 0 (
     %PIP_EXTRA_ARGS% ^
     %CFG_REQUIREMENTS%
 
+
+@rem ################################
+:create_bin_junction
 @rem # Create junction to bin to have the same directory between linux and windows
 if exist "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" (
     rmdir /s /q "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin"
@@ -171,7 +178,6 @@ exit /b 0
 
 
 @rem ################################
-
 :cli_help
     echo An initial configuration script
     echo "  usage: configure [options]"
@@ -195,11 +201,12 @@ exit /b 0
     exit /b 0
 
 
+@rem ################################
 :clean
 @rem # Remove cleanable file and directories and files from the root dir.
 echo "* Cleaning ..."
 for %%F in (%CLEANABLE%) do (
     rmdir /s /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1
     del /f /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1
 )
-exit /b 0
+exit /b 0