move wheel and sdist out of container

Author: gilesknap

.dockerignore

@@ -1,5 +1,4 @@
 build/
-dist/
 .mypy_cache
 .tox
 .venv*

.github/workflows/code.yml

@@ -6,6 +6,10 @@ on:
   schedule:
     # Run every Monday at 8am to check latest versions of dependencies
     - cron: "0 8 * * WED"
+env:
+  # the target python version. Make sure the Dockerfile uses the same
+  # version for its build and runtime targets.
+  PYTHON: "3.10"
 
 jobs:
   lint:
@@ -20,7 +24,7 @@ jobs:
       - name: Setup python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: ${{env.PYTHON}}
 
       - name: Lint
         run: |
@@ -53,7 +57,10 @@ jobs:
           python-version: ${{ matrix.python }}
 
       - name: Install with latest dependencies
-        run: pip install .[dev]
+        run: |
+          pip install .[dev]
+          mkdir -p lockfiles
+          pip freeze > lockfiles/requirements-dev-py${{ matrix.python }}.txt
 
       - name: Run tests
         run: pytest tests
@@ -64,9 +71,62 @@ jobs:
           name: ${{ matrix.python }}/${{ matrix.os }}
           files: cov.xml
 
+      - name: Upload lock files
+        uses: actions/upload-artifact@v3
+        with:
+          name: lockfiles
+          path: lockfiles
+
+  dist:
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
+    runs-on: "ubuntu-latest"
+
+    steps:
+      - name: Checkout Source
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{env.PYTHON}}
+
+      - name: Build Sdist and wheel
+        run: |
+          export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \
+          pipx run --python $(which python${{env.PYTHON}}) build
+
+      # ${GITHUB_REPOSITORY##*/} is the repo name without org
+      # Replace this with the cli command if different to the repo name
+      - name: Test cli works in sdist installed in local python
+        run: |
+          touch requirements.txt
+          pip install -r requirements.txt dist/*.gz ${GITHUB_REPOSITORY##*/} --version
+
+      # create a requirements.txt to be published as a github release asset
+      - name: get a requirements.txt from the installed sdist
+        run: |
+          mkdir -p lockfiles
+          pip freeze > lockfiles/requirements.txt
+
+      - name: Upload sdist and wheel as artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: dist
+          path: dist
+
+      - name: Upload lock files
+        uses: actions/upload-artifact@v3
+        with:
+          name: lockfiles
+          path: lockfiles
+
   container:
+    needs: [dist]
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
     runs-on: ubuntu-latest
+
     permissions:
       contents: read
       packages: write
@@ -77,6 +137,12 @@ jobs:
         with:
           fetch-depth: 0
 
+      - uses: actions/download-artifact@v3
+
+      - name: PrepareReg Names
+        run: |
+          echo IMAGE_REPOSITORY=ghcr.io/$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV
+
       - name: Log in to GitHub Docker Registry
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v2
@@ -89,74 +155,43 @@ jobs:
         id: meta
         uses: docker/metadata-action@v4
         with:
-          images: ghcr.io/${{ github.repository }}
+          images: ${{env.IMAGE_REPOSITORY}}
           tags: |
-            type=ref,event=branch
             type=ref,event=tag
+            type=raw,value=latest
 
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v2
 
-      - name: Build developer image for testing
-        uses: docker/build-push-action@v3
-        with:
-          tags: build:latest
-          context: .
-          target: build
-          load: true
-
-      - name: Run tests in the container locked with requirements_dev.txt
-        run: |
-          docker run --name test build bash /project/.github/workflows/container_tests.sh
-          docker cp test:/project/dist .
-          docker cp test:/project/lockfiles .
-          docker cp test:/project/cov.xml .
-
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
-        with:
-          name: 3.10-locked/ubuntu-latest
-          files: cov.xml
-
       - name: Build runtime image
         uses: docker/build-push-action@v3
         with:
-          push: ${{ github.event_name != 'pull_request' }}
+          push: true
+          load: true
           tags: ${{ steps.meta.outputs.tags }}
           context: .
-          labels: ${{ steps.meta.outputs.labels }}
 
       - name: Test cli works in runtime image
         # check that the first tag can run with --version parameter
-        run: docker run $(echo ${{ steps.meta.outputs.tags }} | head -1) --version
-
-      - name: Test cli works in sdist installed in local python
-        # ${GITHUB_REPOSITORY##*/} is the repo name without org
-        # Replace this with the cli command if different to the repo name
-        run: pip install dist/*.gz && ${GITHUB_REPOSITORY##*/} --version
-
-      - name: Upload build files
-        uses: actions/upload-artifact@v3
-        with:
-          name: dist
-          path: dist
+        run: docker run ${{env.IMAGE_REPOSITORY}}:latest --version
 
-      - name: Upload lock files
-        uses: actions/upload-artifact@v3
-        with:
-          name: lockfiles
-          path: lockfiles
+      # TODO upload a tar of the image for later publishing
+      # TODO OR do the publish here ??? if its quick enough - MUCH easier ?
 
   release:
     # upload to PyPI and make a release on every tag
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
-    needs: container
+    needs: [lint, dist]
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/download-artifact@v3
 
+      - name: fixup requirements files
+        # use sed to comment out the self references in requirements files
+        run: sed -i '/file:/s/^/# Requirements for /' lockfiles/requirements*.txt
+
       - name: Github Release
         # We pin to the SHA, not the tag, for security reasons.
         # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions

Dockerfile

@@ -8,36 +8,21 @@ FROM python:3.10 as build
 # Add any system dependencies for the developer/build environment here
 RUN apt-get update && apt-get upgrade -y && \
     apt-get install -y --no-install-recommends \
-    build-essential \
     busybox \
-    git \
-    net-tools \
-    vim \
     && rm -rf /var/lib/apt/lists/* \
     && busybox --install
 
 COPY . /project
 WORKDIR /project
 
-# make the wheel outside of the venv so 'build' does not dirty requirements.txt
-RUN pip install --upgrade pip build && \
-    export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \
-    git diff && \
-    python -m build && \
-    touch requirements.txt
-
 # set up a virtual environment and put it in PATH
 RUN python -m venv /venv
 ENV PATH=/venv/bin:$PATH
 
-# install the wheel and generate the requirements file
+# install the wheel
 RUN pip install --upgrade pip && \
-    pip install -r requirements.txt dist/*.whl && \
-    mkdir -p lockfiles && \
-    pip freeze  > lockfiles/requirements.txt && \
-    # we don't want to include our own wheel in requirements - remove with sed
-    # and replace with a comment to avoid a zero length asset upload later
-    sed -i '/file:/s/^/# Requirements for /' lockfiles/requirements.txt
+    touch requirements.txt && \
+    pip install -r requirements.txt dist/*.whl
 
 FROM python:3.10-slim as runtime