Merge branch 'review-nov22' of github.com:DiamondLightSource/python3-pip-skeleton into review-nov22

gilesknap · gilesknap · commit dfaf08837216 · 2022-11-09T12:34:30.000Z
diff --git a/.dockerignore b/.dockerignore
@@ -1,5 +1,4 @@
 build/
-dist/
 .mypy_cache
 .tox
 .venv*
diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml
@@ -20,21 +20,29 @@ jobs:
       - name: Setup python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.x"
 
       - name: Lint
         run: |
-          touch requirements_dev.txt requirements.txt
-          pip install -r requirements.txt -r requirements_dev.txt -e .[dev]
+          touch requirements-lint.txt requirements.txt
+          pip install -r requirements.txt -r requirements-lint.txt -e .[dev]
           tox -e pre-commit,mypy
+          mkdir -p lockfiles
+          pip freeze > lockfiles/requirements-lint.txt
+
+      - name: Upload lockfiles
+        uses: actions/upload-artifact@v3
+        with:
+          name: lockfiles
+          path: lockfiles
 
   test:
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
     strategy:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest"] # can add windows-latest, macos-latest
-        python: ["3.8", "3.9", "3.10"]
+        python: ["3.8", "3.9", "3.10", "3.11"]
 
     runs-on: ${{ matrix.os }}
     env:
@@ -58,7 +66,10 @@ jobs:
           python-version: ${{ matrix.python }}
 
       - name: Install with latest dependencies
-        run: pip install .[dev]
+        run: |
+          pip install .[dev]
+          mkdir -p lockfiles
+          pip freeze > lockfiles/requirements-test-${{ matrix.python }}-${{ matrix.os }}.txt
 
       - name: Run tests
         run: pytest tests
@@ -69,18 +80,65 @@ jobs:
           name: ${{ matrix.python }}/${{ matrix.os }}
           files: cov.xml
 
-  container:
+      - name: Upload lockfiles
+        uses: actions/upload-artifact@v3
+        with:
+          name: lockfiles
+          path: lockfiles
+
+  dist:
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
+    runs-on: "ubuntu-latest"
+
+    steps:
+      - name: Checkout Source
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Build Sdist and wheel
+        run: |
+          export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \
+          pipx run build
+
+      # ${GITHUB_REPOSITORY##*/} is the repo name without org
+      # Replace this with the cli command if different to the repo name
+      - name: Test cli works using the wheel installed in local python
+        run: |
+          touch requirements.txt
+          pip install -r requirements.txt dist/*.whl
+          ${GITHUB_REPOSITORY##*/} --version
+
+      - name: Test Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: pipx run twine check dist/*
+
+      - name: Upload sdist and wheel as artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: dist
+          path: dist
+
+  container:
+    needs: [lint, dist, test]
     runs-on: ubuntu-latest
+
     permissions:
       contents: read
       packages: write
 
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
+
+      # image names must be all lower case
+      - run: |
+          echo IMAGE_REPOSITORY=ghcr.io/$(tr '[:upper:]' '[:lower:]' <<< "${{ github.repository }}") >> $GITHUB_ENV
+
+      # obtain the python wheel from the dist step
+      - uses: actions/download-artifact@v3
 
       - name: Log in to GitHub Docker Registry
         if: github.event_name != 'pull_request'
@@ -94,75 +152,51 @@ jobs:
         id: meta
         uses: docker/metadata-action@v4
         with:
-          images: ghcr.io/${{ github.repository }}
+          images: ${{ env.IMAGE_REPOSITORY }}
           tags: |
-            type=ref,event=branch
             type=ref,event=tag
+            type=raw,value=latest
 
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v2
 
-      - name: Build developer image for testing
-        uses: docker/build-push-action@v3
-        with:
-          tags: build:latest
-          context: .
-          target: build
-          load: true
-
-      - name: Run tests in the container locked with requirements_dev.txt
-        run: |
-          docker run --name test build bash /project/.github/workflows/container_tests.sh
-          docker cp test:/project/dist .
-          docker cp test:/project/lockfiles .
-          docker cp test:/project/cov.xml .
-
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
-        with:
-          name: 3.10-locked/ubuntu-latest
-          files: cov.xml
-
       - name: Build runtime image
         uses: docker/build-push-action@v3
         with:
-          push: ${{ github.event_name != 'pull_request' }}
+          push: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }}
+          load: ${{ ! (github.event_name == 'push' && startsWith(github.ref, 'refs/tags')) }}
           tags: ${{ steps.meta.outputs.tags }}
           context: .
-          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
       - name: Test cli works in runtime image
-        # check that the first tag can run with --version parameter
-        run: docker run $(echo ${{ steps.meta.outputs.tags }} | head -1) --version
-
-      - name: Test cli works in sdist installed in local python
-        # ${GITHUB_REPOSITORY##*/} is the repo name without org
-        # Replace this with the cli command if different to the repo name
-        # (python3-pip-skeleton-cli replaces this with python3-pip-skeleton)
-        run: pip install dist/*.gz && python3-pip-skeleton --version
-
-      - name: Upload build files
-        uses: actions/upload-artifact@v3
-        with:
-          name: dist
-          path: dist
+        # check that the latest tag can run with --version parameter
+        run: |
+          docker run ${{ env.IMAGE_REPOSITORY }} --version
+          mkdir -p lockfiles
+          docker run --entrypoint pip ${{ env.IMAGE_REPOSITORY }} freeze > lockfiles/requirements.txt
 
-      - name: Upload lock files
+      - name: Upload lockfiles
         uses: actions/upload-artifact@v3
         with:
           name: lockfiles
           path: lockfiles
 
   release:
     # upload to PyPI and make a release on every tag
-    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
-    needs: container
+    needs: [lint, dist, test]
+    if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }}
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/download-artifact@v3
 
+      - name: fixup requirements files
+        # use sed to comment out the self references in requirements files
+        run: sed -i '/file:/s/^/# Requirements for /' lockfiles/requirements*.txt
+
       - name: Github Release
         # We pin to the SHA, not the tag, for security reasons.
         # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions
diff --git a/.github/workflows/container_tests.sh b/.github/workflows/container_tests.sh
diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml
@@ -1,6 +1,7 @@
 name: Link Check
 
 on:
+  workflow_dispatch:
   schedule:
     # Run every Monday at 8am to check URL links still resolve
     - cron: "0 8 * * WED"
@@ -31,4 +32,4 @@ jobs:
           pip install -r requirements_dev.txt -e .[dev]
 
       - name: Check links
-        run: tox -e docs -- -b linkcheck
+        run: tox -e docs build -- -b linkcheck
diff --git a/Dockerfile b/Dockerfile
@@ -3,45 +3,27 @@
 # The devcontainer should use the build target and run as root with podman
 # or docker with user namespaces.
 #
-FROM python:3.10 as build
-
-# Add any system dependencies for the developer/build environment here
-RUN apt-get update && apt-get upgrade -y && \
-    apt-get install -y --no-install-recommends \
-    build-essential \
-    busybox \
-    git \
-    graphviz \
-    net-tools \
-    vim \
-    && rm -rf /var/lib/apt/lists/* \
-    && busybox --install
+FROM python:3.11 as build
+
+# Add any system dependencies for the developer/build environment here e.g.
+# RUN apt-get update && apt-get upgrade -y && \
+#     apt-get install -y --no-install-recommends \
+#     busybox \
+#     && rm -rf /var/lib/apt/lists/* \
+#     && busybox --install
 
 COPY . /project
 WORKDIR /project
 
-# make the wheel outside of the venv so 'build' does not dirty requirements.txt
-RUN pip install --upgrade pip build && \
-    export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \
-    git diff && \
-    python -m build && \
-    touch requirements.txt
-
 # set up a virtual environment and put it in PATH
 RUN python -m venv /venv
 ENV PATH=/venv/bin:$PATH
-ENV TOX_DIRECT=1
-
-# install the wheel and generate the requirements file
-RUN pip install --upgrade pip && \
-    pip install -r requirements.txt dist/*.whl && \
-    mkdir -p lockfiles && \
-    pip freeze  > lockfiles/requirements.txt && \
-    # we don't want to include our own wheel in requirements - remove with sed
-    # and replace with a comment to avoid a zero length asset upload later
-    sed -i '/file:/s/^/# Requirements for /' lockfiles/requirements.txt
-
-FROM python:3.10-slim as runtime
+
+# install the wheel
+RUN touch requirements.txt && \
+    pip install -r requirements.txt dist/*.whl
+
+FROM python:3.11-slim as runtime
 
 # Add apt-get system dependecies for runtime here if needed
 
diff --git a/setup.cfg b/setup.cfg
@@ -106,24 +106,21 @@ source =
 # NOTE that we pre-install all tools in the dev dependencies (including tox).
 # Hence the use of allowlist_externals instead of using the tox virtualenvs.
 # This ensures a match between developer time tools in the IDE and tox tools.
-# Setting TOX_DIRECT=1 in the environment will make this even faster
 [tox:tox]
 skipsdist = True
 
-[testenv:pytest]
-allowlist_externals = pytest
-commands = pytest {posargs}
-
-[testenv:mypy]
-allowlist_externals = mypy
-commands = mypy src tests {posargs}
-
-[testenv:pre-commit]
-allowlist_externals = pre-commit
-commands = pre-commit run --all-files {posargs}
-
-[testenv:docs]
-allowlist_externals =
+[testenv:{pre-commit,mypy,pytest,docs}]
+# Don't create a virtualenv for the command, requires tox-direct plugin
+direct = True
+passenv = *
+allowlist_externals = 
+    pytest 
+    pre-commit
+    mypy
     sphinx-build
     sphinx-autobuild
-commands = sphinx-{posargs:build -EW --keep-going} -T docs build/html
+commands =
+    pytest: pytest {posargs}
+    mypy: mypy src tests {posargs}
+    pre-commit: pre-commit run --all-files {posargs}
+    docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html

Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,4 @@`
`1`	`1`	`build/`
`2`		`-dist/`
`3`	`2`	`.mypy_cache`
`4`	`3`	`.tox`
`5`	`4`	`.venv*`