[DEVOPS-1591] Add CI configuration for building images (#11)

* Fix permissions on all files, add pre-commit configuration

* Run pre-commit checks in github actions

* Add docker bake file

* Fix task configurations

* Remove 3.11 for testing

* Don't fail fast either

* Quote versions

* Replace hadolint

* Add additional copies for R base image

* Fix build targets for GPU

* Remove GPU builds for now

* Fix parsing of environment variable

* Fix target processing

* Use intermediate step to generate the target

* Fix job name

* Fix quoting

* Prefix target with language

* Fix rlang target

* Set the appropriate targets for various builds

* Remove 'main' target from rlang

* Test pushing python kernels from pull request

* Add labels

* Add GPU builds

* Fix variable usage

* Syntax correction

* Fix some more quoting

* Fix if

* Set base image

* Fix bake contexts for GPU images

* Also need to copy base files

* Fix quoting

* Copy GPU files all the time for testing

* Use 8CPU runner group

* Use public runner group

* Rerun checks

* Use github.ref_name for the tag

* Fix quoting

* Configure dynamic tags

* Fix the tag variable syntax

* Fix linting

* Use script for setting TAG and TIMESTAMP

* Add context to TODO comment

* Fix typo

* Use 1.5 Dockerfile syntax

* Pin 3rd party workflows

* Provide comments and links for description of bake configuration

* Push images for another test

* Add -base suffix to R base image

* Disable PR builds again

* Push some latest tags for testing

* Revert builds to not use latest, or build on PRs

Author: robwittman

.github/workflows/build.yml

@@ -49,3 +49,96 @@ jobs:
 
       - name: Run pre-commit hooks ✅
         run: pre-commit run --all-files --hook-stage manual
+
+  build_python_kernels:
+    runs-on:
+      group: ubuntu-22.04-8cpu-public
+    permissions:
+      id-token: write
+      packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ["3.9", "3.10"] # TODO: Add 3.11 after fixing dependency conflicts
+        identifier: [base, base-gpu]
+    env:
+      VERSION: ${{ matrix.version }}
+      TARGET: python_${VERSION//./_}
+      BUILD_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      GITHUB_SHA: ${{ github.sha }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Task
+        uses: arduino/setup-task@e26d8975574116b0097a1161e0fe16ba75d84c1c # v1.0.3
+        with:
+          repo-token: ${{ github.token }}
+      - run: |
+          task python:base:copy-files IDENTIFIER="base" NBL_PYTHON_VERSION=${{ matrix.version }}
+          if [[ "${{ matrix.identifier }}" == "base-gpu" ]];
+          then
+            task python:base:copy-files IDENTIFIER=${{ matrix.identifier }} NBL_PYTHON_VERSION=${{ matrix.version }}
+          fi
+          task python:noteable:copy-files IDENTIFIER=${{ matrix.identifier }} NBL_PYTHON_VERSION=${{ matrix.version }}
+      - run: scripts/set-variables.sh ${{ github.ref_name }} ${{ github.event_name }} ${{ github.event.pull_request.number }}
+      - uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+      - name: Set target
+        id: target
+        run: |
+          if [[ "${{ matrix.identifier }}" == "base" ]]
+          then
+            target="python_${VERSION//./_}"
+          else
+            target="python_${VERSION//./_}_gpu"
+          fi
+          echo "TARGET=${target}" >> "$GITHUB_OUTPUT"
+
+      - uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v2.9.1
+
+      - name: Build python kernels
+        env:
+        uses: docker/bake-action@f32f8b8d70bc284af19f8148dd14ad1d2fbc6c28 # v3.1.0
+        with:
+          push: ${{ github.event_name != 'pull_request' }}
+          targets: ${{ steps.target.outputs.TARGET }}
+
+  build_rlang_kernels:
+    runs-on: ubuntu-22.04
+    permissions:
+      id-token: write
+      packages: write
+    strategy:
+      matrix:
+        version: [4.3.0]
+    env:
+      VERSION: ${{ matrix.version }}
+      BUILD_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      GITHUB_SHA: ${{ github.ref }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Task
+        uses: arduino/setup-task@e26d8975574116b0097a1161e0fe16ba75d84c1c # v1.0.3
+        with:
+          repo-token: ${{ github.token }}
+      - run: |
+          task r:base:copy-files NBL_LANGUAGE_VERSION=${{ matrix.version }}
+      - run: scripts/set-variables.sh ${{ github.ref_name }} ${{ github.event_name }} ${{ github.event.pull_request.number }}
+      - uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+      - name: Set target
+        id: target
+        run: echo "TARGET=rlang_${VERSION//./_}" >> "$GITHUB_OUTPUT"
+
+      - uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v2.9.1
+
+      - name: Build kernels
+        uses: docker/bake-action@f32f8b8d70bc284af19f8148dd14ad1d2fbc6c28 # v3.1.0
+        with:
+          push: ${{ github.event_name != 'pull_request' }}
+          targets: ${{ steps.target.outputs.TARGET }}

.gitignore

@@ -1,3 +1,6 @@
+# Development files
+.idea
+
 # ignore these everywhere
 .pythonrc
 .Rprofile

R/base/4.3.0/Dockerfile

@@ -1,4 +1,4 @@
-# syntax = docker/dockerfile:1.2.1
+# syntax = docker/dockerfile:1.5
 # ---
 # Bare minimum R 4.3.x image with IRkernel installed
 # - no R packages aside from builtins and IRkernel
@@ -62,15 +62,3 @@ EXPOSE 50001-50005
 
 ENTRYPOINT ["/tini", "-g", "--"]
 CMD ["run.sh"]
-
-ARG NBL_ARG_BUILD_TIMESTAMP="undefined"
-ARG NBL_ARG_REVISION="undefined"
-ARG NBL_ARG_BUILD_URL="undefined"
-ARG NBL_ARG_VERSION="undefined"
-LABEL org.opencontainers.image.created="${NBL_ARG_BUILD_TIMESTAMP}" \
-  org.opencontainers.image.revision="${NBL_ARG_REVISION}" \
-  org.opencontainers.image.source="https://github.com/noteable-io/polymorph" \
-  org.opencontainers.image.title="noteable-R-${NBL_R_VERSION}" \
-  org.opencontainers.image.url="${NBL_ARG_BUILD_URL}" \
-  org.opencontainers.image.vendor="Noteable" \
-  org.opencontainers.image.version="${NBL_ARG_VERSION}"

R/noteable/4.3.0/Dockerfile

@@ -1,9 +1,8 @@
-# syntax = docker/dockerfile:1.2.1
+# syntax = docker/dockerfile:1.5
 # Noteable build: adds packages to enable Noteable-specific functionality:
 # - DEX support (via .Rprofile)
-ARG BASE_IMAGE
 # hadolint ignore=DL3006
-FROM ${BASE_IMAGE} as base
+FROM base
 
 USER noteable
 

Taskfile.R.yaml

@@ -23,7 +23,8 @@ tasks:
     cmds:
       - task copy-root-files LANGUAGE=R IDENTIFIER={{.IDENTIFIER}} NBL_LANGUAGE_VERSION={{.NBL_R_VERSION}}
       - task copy-language-files LANGUAGE=R IDENTIFIER={{.IDENTIFIER}} NBL_LANGUAGE_VERSION={{.NBL_R_VERSION}}
-
+      - cp R/noteable/.Rprofile R/noteable/{{.NBL_R_VERSION}}/.Rprofile
+      - cp R/noteable/requirements.R R/noteable/{{.NBL_R_VERSION}}/requirements.R
   base:build:
     desc: Build the R 4.x base image after copying required files
     cmds:

Taskfile.python.yaml

@@ -29,6 +29,10 @@ tasks:
     cmds:
       - task copy-root-files LANGUAGE=python IDENTIFIER={{.IDENTIFIER}} NBL_LANGUAGE_VERSION={{.NBL_PYTHON_VERSION}}
       - task copy-language-files LANGUAGE=python IDENTIFIER={{.IDENTIFIER}} NBL_LANGUAGE_VERSION={{.NBL_PYTHON_VERSION}}
+      - cp python/base-gpu/environment.txt python/base-gpu/{{.NBL_PYTHON_VERSION}}/environment.txt
+      - cp python/base-gpu/gpu.Aptfile python/base-gpu/{{.NBL_PYTHON_VERSION}}/gpu.Aptfile
+      - cp python/base-gpu/run.sh python/base-gpu/{{.NBL_PYTHON_VERSION}}/run.sh
+      - cp python/base-gpu/initial-condarc python/base-gpu/{{.NBL_PYTHON_VERSION}}/initial-condarc
 
   base:pyenv:install:
     desc: Install the specified version of Python using pyenv