Build images

Author: manics

.github/workflows/build.yml

@@ -1,91 +1,90 @@
-# This is a GitHub workflow defining a set of jobs with a set of steps.
-# ref: https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions
-#
-# Test build release artifacts (PyPI package, Docker images) and publish them on
-# pushed git tags.
-#
-name: Release
+name: Build
 
 on:
   pull_request:
     paths-ignore:
       - "docs/**"
-      - "**.md"
-      - "**.rst"
-      - ".github/workflows/*"
-      - "!.github/workflows/release.yml"
   push:
     paths-ignore:
       - "docs/**"
-      - "**.md"
-      - "**.rst"
-      - ".github/workflows/*"
-      - "!.github/workflows/release.yml"
     branches-ignore:
       - "dependabot/**"
       - "pre-commit-ci-update-config"
-    tags:
-      - "**"
   workflow_dispatch:
 
+# Tags are calculated based on the existing tags in the repository
+# so we can only run builds consecutively, otherwise two images in
+# different builds may have the same tag.
+# If there are multiple pending jobs older pending jobs will be cancelled.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+env:
+  # If we're on this branch don't bother fetching every tag since we know
+  # this will be the most recent tag, i.e. builds on this branch should always
+  # update the `latest` tag
+  LATEST_BRANCH: main
+  # Only push images if this workflow is run on this branch
+  # This ensures if a a new branch is created in this repo it won't automatically
+  # push an image.
+  # If this is a backport then change this to the name of the backports branch
+  PUBLISH_BRANCH: main
+
+  # IMAGE: jupyterhub/jupyterhub
+  IMAGE: manics/jupyterhub-image-test
+  SINGLEUSER: manics/jupyterhub-singleuser-image-test
+  PUBLISH_DOCKERIO: "false"
+
+  # Enable caching across builds, set to "" to disable
+  CACHE_FROM: type=gha
+  CACHE_TO: type=gha,mode=max
+
 jobs:
-  build-release:
+  tag:
     runs-on: ubuntu-22.04
+    timeout-minutes: 2
+    outputs:
+      existing-tags: ${{ steps.quayio.outputs.tags }}
+      new-tag: ${{ steps.new.outputs.TAG }}
+      jupyterhub-version: ${{ steps.version.outputs.VERSION }}
+
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
-          cache: pip
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version: "20"
-
-      - name: install build requirements
-        run: |
-          npm install -g yarn
-          pip install --upgrade pip
-          pip install build
-          pip freeze
-
-      - name: build release
-        run: |
-          python -m build --sdist --wheel .
-          ls -l dist
-
-      - name: verify sdist
-        run: |
-          ./ci/check_sdist.py dist/jupyterhub-*.tar.gz
-
-      - name: verify data-files are installed where they are found
-        run: |
-          pip install dist/*.whl
-          ./ci/check_installed_data.py
 
-      - name: verify sdist can be installed without npm/yarn
+      - name: JupyterHub version
+        id: version
         run: |
-          docker run --rm -v $PWD/dist:/dist:ro docker.io/library/python:3.9-slim-bullseye bash -c 'pip install /dist/jupyterhub-*.tar.gz'
+          VERSION=$(grep '^jupyterhub==' base/requirements.txt | cut -d= -f3)
+          if [[ $VERSION =~ [0-9]+\.[0-9]+\.[0-9]+ ]]; then
+            echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+          else
+            echo "Failed to get JupyterHub version"
+            exit 1
+          fi
 
-      # ref: https://github.com/actions/upload-artifact#readme
-      - uses: actions/upload-artifact@v4
+      - name: Get build-number by looking at existing tags
+        id: quayio
+        uses: manics/action-get-quayio-tags@main
         with:
-          name: jupyterhub-${{ github.sha }}
-          path: "dist/*"
-          if-no-files-found: error
-
-      - name: Publish to PyPI
-        if: startsWith(github.ref, 'refs/tags/')
-        env:
-          TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+          repository: ${{ env.IMAGE }}
+          version: ${{ steps.version.outputs.VERSION }}
+          # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/accessing-contextual-information-about-workflow-runs#github-context
+          # If this is not LATEST_BRANCH, nor is it a pull request against LATEST_BRANCH
+          # assume it's a backport branch which means we need to get all tags to work out
+          # which MAJOR and MAJOR.MINOR aliases are needed.
+          allTags: ${{ (github.ref != format('refs/heads/{0}', env.LATEST_BRANCH)) && (github.base_ref != format('refs/heads/{0}', env.LATEST_BRANCH)) }}
+          strict: "true"
+
+      - name: Get new tag
+        id: new
         run: |
-          pip install twine
-          twine upload --skip-existing dist/*
+          echo "TAG=${{ steps.version.outputs.VERSION }}-${{ steps.quayio.outputs.buildNumber }}" >> $GITHUB_OUTPUT
 
   publish-docker:
     runs-on: ubuntu-22.04
     timeout-minutes: 30
+    needs:
+      - tag
 
     services:
       # So that we can test this in PRs/branches
@@ -95,19 +94,23 @@ jobs:
           - 5000:5000
 
     steps:
+      - uses: actions/checkout@v4
+
       - name: Should we push this image to a public registry?
         run: |
-          if [ "${{ startsWith(github.ref, 'refs/tags/') || (github.ref == 'refs/heads/main') }}" = "true" ]; then
+          if [ "${{ github.ref == format('refs/heads/{0}', env.PUBLISH_BRANCH) }}" = "true" ]; then
               echo "REGISTRY=quay.io/" >> $GITHUB_ENV
+              echo "PUBLIC=true" >> $GITHUB_ENV
           else
               echo "REGISTRY=localhost:5000/" >> $GITHUB_ENV
+              echo "PUBLIC=false" >> $GITHUB_ENV
           fi
 
       - uses: actions/checkout@v4
 
       # Setup docker to build for multiple platforms, see:
-      # https://github.com/docker/build-push-action/tree/v2.4.0#usage
-      # https://github.com/docker/build-push-action/blob/v2.4.0/docs/advanced/multi-platform.md
+      # https://github.com/docker/build-push-action/tree/v6.9.0?tab=readme-ov-file#usage
+      # https://docs.docker.com/build/ci/github-actions/multi-platform/
       - name: Set up QEMU (for docker buildx)
         uses: docker/setup-qemu-action@v3
 
@@ -123,53 +126,59 @@ jobs:
         # .  Quay.io org
         # 2. Giving it enough permissions to push to the jupyterhub and singleuser images
         # 3. Putting the robot account's username and password in GitHub actions environment
-        if: env.REGISTRY != 'localhost:5000/'
+        if: env.PUBLIC == 'true'
         run: |
           docker login -u "${{ secrets.QUAY_USERNAME }}" -p "${{ secrets.QUAY_PASSWORD }}" "${{ env.REGISTRY }}"
-          docker login -u "${{ secrets.DOCKERHUB_USERNAME }}" -p "${{ secrets.DOCKERHUB_TOKEN }}" docker.io
+          if [ "${{ env.PUBLISH_DOCKERIO }}" = "true" ]; then
+            docker login -u "${{ secrets.DOCKERHUB_USERNAME }}" -p "${{ secrets.DOCKERHUB_TOKEN }}" docker.io
+          fi
 
-      # image: jupyterhub/jupyterhub
+      # image: env.IMAGE
       #
       # https://github.com/jupyterhub/action-major-minor-tag-calculator
       # If this is a tagged build this will return additional parent tags.
       # E.g. 1.2.3 is expanded to Docker tags
       # [{prefix}:1.2.3, {prefix}:1.2, {prefix}:1, {prefix}:latest] unless
       # this is a backported tag in which case the newer tags aren't updated.
-      # For branches this will return the branch name.
-      # If GITHUB_TOKEN isn't available (e.g. in PRs) returns no tags [].
-      - name: Get list of jupyterhub tags
+      - name: Calculate tags
         id: jupyterhubtags
-        uses: jupyterhub/action-major-minor-tag-calculator@v3
+        uses: manics/action-major-minor-tag-calculator@allow-external-tag
         with:
-          githubToken: ${{ secrets.GITHUB_TOKEN }}
+          tagList: ${{ needs.tag.outputs.existing-tags }}
+          currentTag: ${{ needs.tag.outputs.new-tag }}
           prefix: >-
-            ${{ env.REGISTRY }}jupyterhub/jupyterhub:
-            jupyterhub/jupyterhub:
-          defaultTag: "${{ env.REGISTRY }}jupyterhub/jupyterhub:noref"
-          branchRegex: ^\w[\w-.]*$
+            ${{ env.REGISTRY }}${{ env.IMAGE }}:
+            ${{ (env.PUBLIC == 'true' && env.PUBLISH_DOCKERIO == 'true') && format('{0}:', env.IMAGE) || '' }}
+
+      - name: Print tags
+        run: |
+          echo "Existing tags: ${{ needs.tag.outputs.existing-tags }}"
+          echo "New tags: ${{ needs.tag.outputs.new-tag }}"
+          echo "Image tags: ${{ steps.jupyterhubtags.outputs.tags }}"
 
       - name: Build and push jupyterhub
         uses: docker/build-push-action@v6
         with:
-          context: .
+          context: base
           platforms: linux/amd64,linux/arm64
           push: true
           # tags parameter must be a string input so convert `gettags` JSON
           # array into a comma separated list of tags
           tags: ${{ join(fromJson(steps.jupyterhubtags.outputs.tags)) }}
+          cache-from: ${{ env.CACHE_FROM }}
+          cache-to: ${{ env.CACHE_TO }}
 
-      # image: jupyterhub/jupyterhub-onbuild
+      # image: env.IMAGE-onbuild
       #
       - name: Get list of jupyterhub-onbuild tags
         id: onbuildtags
-        uses: jupyterhub/action-major-minor-tag-calculator@v3
+        uses: manics/action-major-minor-tag-calculator@allow-external-tag
         with:
-          githubToken: ${{ secrets.GITHUB_TOKEN }}
+          tagList: ${{ needs.tag.outputs.existing-tags }}
+          currentTag: ${{ needs.tag.outputs.new-tag }}
           prefix: >-
-            ${{ env.REGISTRY }}jupyterhub/jupyterhub-onbuild:
-            jupyterhub/jupyterhub-onbuild:
-          defaultTag: "${{ env.REGISTRY }}jupyterhub/jupyterhub-onbuild:noref"
-          branchRegex: ^\w[\w-.]*$
+            ${{ env.REGISTRY }}${{ env.IMAGE }}-onbuild:
+            ${{ (env.PUBLIC == 'true' && env.PUBLISH_DOCKERIO == 'true') && format('{0}-onbuild:', env.IMAGE) || '' }}
 
       - name: Build and push jupyterhub-onbuild
         uses: docker/build-push-action@v6
@@ -180,19 +189,20 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ join(fromJson(steps.onbuildtags.outputs.tags)) }}
+          cache-from: ${{ env.CACHE_FROM }}
+          cache-to: ${{ env.CACHE_TO }}
 
-      # image: jupyterhub/jupyterhub-demo
+      # image: env.IMAGE-demo
       #
       - name: Get list of jupyterhub-demo tags
         id: demotags
-        uses: jupyterhub/action-major-minor-tag-calculator@v3
+        uses: manics/action-major-minor-tag-calculator@allow-external-tag
         with:
-          githubToken: ${{ secrets.GITHUB_TOKEN }}
+          tagList: ${{ needs.tag.outputs.existing-tags }}
+          currentTag: ${{ needs.tag.outputs.new-tag }}
           prefix: >-
-            ${{ env.REGISTRY }}jupyterhub/jupyterhub-demo:
-            jupyterhub/jupyterhub-demo:
-          defaultTag: "${{ env.REGISTRY }}jupyterhub/jupyterhub-demo:noref"
-          branchRegex: ^\w[\w-.]*$
+            ${{ env.REGISTRY }}${{ env.IMAGE }}-demo:
+            ${{ (env.PUBLIC == 'true' && env.PUBLISH_DOCKERIO == 'true') && format('{0}-demo:', env.IMAGE) || '' }}
 
       - name: Build and push jupyterhub-demo
         uses: docker/build-push-action@v6
@@ -206,26 +216,29 @@ jobs:
           platforms: linux/amd64
           push: true
           tags: ${{ join(fromJson(steps.demotags.outputs.tags)) }}
+          cache-from: ${{ env.CACHE_FROM }}
+          cache-to: ${{ env.CACHE_TO }}
 
-      # image: jupyterhub/singleuser
+      # image: env.SINGLEUSER
       #
       - name: Get list of jupyterhub/singleuser tags
         id: singleusertags
-        uses: jupyterhub/action-major-minor-tag-calculator@v3
+        uses: manics/action-major-minor-tag-calculator@allow-external-tag
         with:
-          githubToken: ${{ secrets.GITHUB_TOKEN }}
+          tagList: ${{ needs.tag.outputs.existing-tags }}
+          currentTag: ${{ needs.tag.outputs.new-tag }}
           prefix: >-
-            ${{ env.REGISTRY }}jupyterhub/singleuser:
-            jupyterhub/singleuser:
-          defaultTag: "${{ env.REGISTRY }}jupyterhub/singleuser:noref"
-          branchRegex: ^\w[\w-.]*$
+            ${{ env.REGISTRY }}${{ env.SINGLEUSER }}:
+            ${{ (env.PUBLIC == 'true' && env.PUBLISH_DOCKERIO == 'true') && format('{0}:', env.SINGLEUSER) || '' }}
 
       - name: Build and push jupyterhub/singleuser
         uses: docker/build-push-action@v6
         with:
           build-args: |
-            JUPYTERHUB_VERSION=${{ github.ref_type == 'tag' && github.ref_name || format('git:{0}', github.sha) }}
+            JUPYTERHUB_VERSION=${{ needs.tag.outputs.jupyterhub-version }}
           context: singleuser
           platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ join(fromJson(steps.singleusertags.outputs.tags)) }}
+          cache-from: ${{ env.CACHE_FROM }}
+          cache-to: ${{ env.CACHE_TO }}