Tidy up

Author: coretl

.devcontainer.json

@@ -35,5 +35,5 @@
     "workspaceMount": "source=${localWorkspaceFolder},target=${localWorkspaceFolder},type=bind",
     "workspaceFolder": "${localWorkspaceFolder}",
     // After the container is created, install the python project in editable form
-    "postCreateCommand": "pip install $([ -f requirements_dev.txt ] && echo -r requirements_dev.txt ) -e .[dev]"
+    "postCreateCommand": "pip install -e .[dev]"
 }

.dockerignore

@@ -1,54 +1,58 @@
-name: "install requirements"
-description: "run pip install with requirements and upload resulting requirements"
+name: Install requirements
+description: Run pip install with requirements and upload resulting requirements
 inputs:
   requirements_file:
-    description: "name of requirements file to use and upload"
+    description: Name of requirements file to use and upload
     required: true
   install_options:
-    description: "parameters to pass to pip install"
+    description: Parameters to pass to pip install
     required: true
   python_version:
-    description: "python version to install"
+    description: Python version to install
     default: "3.x"
 
 runs:
-  using: "composite"
+  using: composite
 
   steps:
     - name: Setup python
       uses: actions/setup-python@v4
       with:
         python-version: ${{ inputs.python_version }}
 
-    - name: Pip Install
+    - name: Pip install
       run: |
         touch ${{ inputs.requirements_file }}
         # -c uses requirements.txt as constraints, see 'Validate requirements file'
         pip install -c ${{ inputs.requirements_file }} ${{ inputs.install_options }}
       shell: bash
 
-    - name: Create Lockfile
+    - name: Create lockfile
       run: |
         mkdir -p lockfiles
         pip freeze --exclude-editable > lockfiles/${{ inputs.requirements_file }}
         # delete the self referencing line
-        sed -i  '/file:/d' lockfiles/${{ inputs.requirements_file }}
+        sed -i '/file:/d' lockfiles/${{ inputs.requirements_file }}
       shell: bash
 
+    - name: Upload lockfiles
+      uses: actions/upload-artifact@v3
+      with:
+        name: lockfiles
+        path: lockfiles
+
     # This eliminates the class of problems where the requirements being given no
     # longer match what the packages themselves dictate. E.g. In the rare instance
     # where I install some-package which used to depend on vulnerable-dependency
     # but now uses good-dependency (despite being nominally the same version)
     # pip will install both if given a requirements file with -r
-    - name: Validate requirements file
-      run: if [ -s ${{ inputs.requirements_file }} ] ; then
-        diff ${{ inputs.requirements_file }} lockfiles/${{ inputs.requirements_file }};
-        echo "requirements files match";
+    - name: If requirements file exists, check it matches pip installed packages
+      run: |
+        if [ -s ${{ inputs.requirements_file }} ]; then
+          if ! diff -u ${{ inputs.requirements_file }} lockfiles/${{ inputs.requirements_file }}; then
+            echo "Error: ${{ inputs.requirements_file }} need the above changes to be exhaustive"
+            exit 1
+          fi
         fi
       shell: bash
 
-    - name: Upload lockfiles
-      uses: actions/upload-artifact@v3
-      with:
-        name: lockfiles
-        path: lockfiles

.github/actions/install_requirements/action.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Install
+      - name: Install python packages
         uses: ./.github/actions/install_requirements
         with:
           requirements_file: requirements-lint.txt
@@ -26,12 +26,6 @@ jobs:
       - name: Lint
         run: tox -e pre-commit,mypy
 
-      - name: Upload lockfiles
-        uses: actions/upload-artifact@v3
-        with:
-          name: lockfiles
-          path: lockfiles
-
   test:
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
     strategy:
@@ -55,17 +49,18 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
         with:
+          # Need this to get version number from last tag
           fetch-depth: 0
 
-      - name: Install
+      - name: Install python packages
         uses: ./.github/actions/install_requirements
         with:
           python_version: ${{ matrix.python }}
           requirements_file: requirements-test-${{ matrix.os }}-${{ matrix.python }}.txt
           install_options: ${{ matrix.install }}
 
       - name: Run tests
-        run: pytest tests
+        run: pytest
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
@@ -78,32 +73,39 @@ jobs:
     runs-on: "ubuntu-latest"
 
     steps:
-      - name: Checkout Source
+      - name: Checkout
         uses: actions/checkout@v3
         with:
+          # Need this to get version number from last tag
           fetch-depth: 0
 
-      - name: Build Sdist and wheel
+      - name: Build sdist and wheel
         run: |
           export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \
           pipx run build
 
-      - name: Test module --version works using the installed wheel
-        run: |
-          touch requirements.txt
-          pip install -r requirements.txt dist/*.whl
-          # If more than one module in src/ replace with module name to test
-          python -m $(ls src | head -1) --version
-
-      - name: Check for packaging errors
-        run: pipx run twine check dist/*
-
       - name: Upload sdist and wheel as artifacts
         uses: actions/upload-artifact@v3
         with:
           name: dist
           path: dist
 
+      - name: Check for packaging errors
+        run: pipx run twine check dist/*
+
+      - name: Install python packages
+        uses: ./.github/actions/install_requirements
+        with:
+          # Must match dockerfile
+          python_version: "3.11"
+          requirements_file: requirements.txt
+          install_options: dist/*.whl
+
+      - name: Test module --version works using the installed wheel
+        run: |
+          # If more than one module in src/ replace with module name to test
+          python -m $(ls src | head -1) --version
+
   container:
     needs: [lint, dist, test]
     runs-on: ubuntu-latest
@@ -120,8 +122,8 @@ jobs:
       - name: Generate image repo name
         run: echo IMAGE_REPOSITORY=ghcr.io/$(tr '[:upper:]' '[:lower:]' <<< "${{ github.repository }}") >> $GITHUB_ENV
 
-      # obtain the python wheel from the dist step
-      - uses: actions/download-artifact@v3
+      - name: Download wheel and lockfiles
+        uses: actions/download-artifact@v3
 
       - name: Log in to GitHub Docker Registry
         if: github.event_name != 'pull_request'
@@ -155,17 +157,7 @@ jobs:
           cache-to: type=gha,mode=max
 
       - name: Test cli works in runtime image
-        run: |
-          docker run ${{ env.IMAGE_REPOSITORY }} --version
-          mkdir -p lockfiles
-          docker run --entrypoint pip ${{ env.IMAGE_REPOSITORY }} freeze | \
-            sed '/file:/s/^/# Requirements for /' > lockfiles/requirements.txt
-
-      - name: Upload lockfiles
-        uses: actions/upload-artifact@v3
-        with:
-          name: lockfiles
-          path: lockfiles
+        run: docker run ${{ env.IMAGE_REPOSITORY }} --version
 
   release:
     # upload to PyPI and make a release on every tag
@@ -176,6 +168,10 @@ jobs:
     steps:
       - uses: actions/download-artifact@v3
 
+      - name: Fixup blank lockfiles
+        # Github release artifacts can't be blank
+        run: for f in lockfiles/*; [ -s $f ] || echo '# No requirements' >> $f; done
+
       - name: Github Release
         # We pin to the SHA, not the tag, for security reasons.
         # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions

.github/workflows/code.yml

@@ -7,36 +7,28 @@ on:
 jobs:
   docs:
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
-    strategy:
-      fail-fast: false
-      matrix:
-        python: ["3.10"]
-
     runs-on: ubuntu-latest
 
     steps:
       - name: Avoid git conflicts when tag and branch pushed at same time
         if: startsWith(github.ref, 'refs/tags')
         run: sleep 60
 
-      - name: Install python version
-        uses: actions/setup-python@v4
+      - name: Checkout
+        uses: actions/checkout@v3
         with:
-          python-version: ${{ matrix.python }}
+          # Need this to get version number from last tag
+          fetch-depth: 0
 
-      - name: Install Packages
+      - name: Install system packages
         # Can delete this if you don't use graphviz in your docs
         run: sudo apt-get install graphviz
 
-      - name: checkout
-        uses: actions/checkout@v3
+      - name: Install python packages
+        uses: ./.github/actions/install_requirements
         with:
-          fetch-depth: 0
-
-      - name: Install dependencies
-        run: |
-          touch requirements_dev.txt
-          pip install -r requirements_dev.txt -e .[dev]
+          requirements_file: requirements-docs.txt
+          install_options: -e .[dev]
 
       - name: Build docs
         run: tox -e docs

.github/workflows/docs.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: checkout
+      - name: Checkout
         uses: actions/checkout@v3
         with:
           ref: gh-pages
@@ -35,7 +35,7 @@ jobs:
 
       - name: update index and push changes
         run: |
-          rm -r ${{ env.DOCS_VERSION }}
+          rm -r $DOCS_VERSION
           python make_switcher.py --remove $DOCS_VERSION ${{ github.repository }} switcher.json
           git config --global user.name 'GitHub Actions Docs Cleanup CI'
           git config --global user.email '[email protected]'

.github/workflows/docs_clean.yml

@@ -11,16 +11,16 @@ FROM python:3.11 as build
 #     desired-packages \
 #     && rm -rf /var/lib/apt/lists/*
 
-COPY . /project
+COPY dist /project/dist
+COPY lockfiles /project/lockfiles
 WORKDIR /project
 
 # set up a virtual environment and put it in PATH
 RUN python -m venv /venv
 ENV PATH=/venv/bin:$PATH
 
 # install the wheel
-RUN touch requirements.txt && \
-    pip install -r requirements.txt dist/*.whl
+RUN pip install -r lockfiles/requirements.txt dist/*.whl
 
 FROM python:3.11-slim as runtime
 

.github/workflows/pip_install.sh

@@ -12,6 +12,7 @@ classifiers =
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
 
 [options]
 python_requires = >=3.8
@@ -50,8 +51,6 @@ dev =
 
 [options.packages.find]
 where = src
-# Don't include our tests directory in the distribution
-exclude = tests
 
 # Specify any package data to be included in the wheel below.
 # [options.package_data]