Container Publishing and Build Paralellism (#215)

Author: bpkroth

.devcontainer/devcontainer.json

@@ -3,16 +3,16 @@
 {
     "name": "Workspace Dockerfile",
 
-    // Sets the run context to one level up instead of the .devcontainer folder.
-    "context": "..",
+    // We have no context for the devcontainer.
+    "context": ".",
 
     // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
-    "dockerFile": "../docker/benchbase/Dockerfile",
+    "dockerFile": "../docker/benchbase/devcontainer/Dockerfile",
     "build": {
-        "target": "devcontainer",
         "args": {
             "--tag": "benchbase-dev:latest"
-        }
+        },
+        "cacheFrom": "benchbase.azurecr.io/benchbase-dev"
     },
 
     // Set *default* container specific settings.json values on container create.
@@ -35,6 +35,10 @@
 
     // Uncomment to use the Docker CLI from inside the container. See https://aka.ms/vscode-remote/samples/docker-from-docker.
     // "mounts": [ "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind" ],
+    // Use the local maven cache.
+    "mounts": [
+        "source=${localEnv:HOME}${localEnv:USERPROFILE}/.m2,target=/home/containeruser/.m2,type=bind,consistency=cached"
+    ],
 
     // Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root.
     "remoteUser": "containeruser"

.dockerignore

@@ -7,14 +7,29 @@
 # The CI jobs are set up as follows:
 # - One job to build and upload artifacts.
 # - One job per DBMS test suite.
+# - One job to build/test the docker images.
+# - One job to publish the docker image.
 
 name: BenchBase (Java with Maven)
 
 on:
   push:
     branches: [ main ]
+    # Generate new docker images on release tags.
+    tags:
+      - 'v*'
   pull_request:
     branches: [ main ]
+  # Run these workflows on a schedule so that docker images are regularly updated for security patches.
+  schedule:
+    - cron: "1 0 * * *"
+  # Give us a button to allow running the workflow on demand for testing.
+  workflow_dispatch:
+    inputs:
+      tags:
+        description: 'Manual Workflow Run'
+        required: false
+        type: string
 
 env:
   POM_VERSION: 2021-SNAPSHOT
@@ -356,10 +371,14 @@ jobs:
           java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlserver/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true
 
   ## ----------------------------------------------------------------------------------
-  ## Docker Build Test
+  ## Docker Build Test Publish
   ## ----------------------------------------------------------------------------------
-  docker-build-test:
+  docker-build-test-publish:
     runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: 1
+      BENCHBASE_PROFILES: 'cockroachdb mariadb mysql postgres spanner phoenix sqlserver sqlite'
+      CONTAINER_REGISTRY_NAME: ${{ secrets.ACR_LOGINURL }}
     services:
       postgres: # https://hub.docker.com/_/postgres
         image: postgres:latest
@@ -381,25 +400,58 @@ jobs:
           PGPASSWORD=password createdb -h localhost -U admin benchbase
       - name: Checkout repo
         uses: actions/checkout@v2
-      - name: Build docker image with all packages and run test suite
+      # https://github.com/actions/cache/blob/master/examples.md#java---maven
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          # Try to use the same key as https://github.com/actions/setup-java uses in the steps above.
+          key: setup-java-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            setup-java-${{ runner.os }}-maven-
+      - name: Build benchbase-dev image
+        run: |
+          ./docker/benchbase/build-dev-image.sh
+      # Note: this script maps the local .m2 cache into the container.
+      - name: Build the benchbase docker image with all profiles using the dev image
         env:
-          DOCKER_BUILDKIT: 1
-          BENCHBASE_PROFILES: 'cockroachdb mariadb mysql postgres spanner phoenix sqlserver'
+          SKIP_TESTS: 'false'
         run: |
-          docker build --build-arg "BENCHBASE_PROFILES=$BENCHBASE_PROFILES" --build-arg TEST_TARGET=test \
-            --build-arg CONTAINERUSER_UID=$UID --build-arg CONTAINERUSER_GID=$UID \
-            -t benchbase:latest -f ./docker/benchbase/Dockerfile --target fullimage .
-      - name: Run basic benchbase test from the docker image against postgres test DB
+          ./docker/benchbase/build-full-image.sh
+      - name: Run a basic test from the docker image against postgres test DB
         env:
             benchmark: noop
         run: |
-          # Adjust the sample config to talk to the container service instead of localhost.
-          cat "config/postgres/sample_${benchmark}_config.xml" | sed -e 's/localhost:5432/postgres:5432/g' > /tmp/config.xml
-          # Lookup the service container's docker network so we can place the benchbase container in it too.
-          docker_network="$(docker ps --filter expose=5432 --format '{{.Networks}}')"
-          # Map the adjusted config into the container and use it to run the test.
-          mkdir -p results
-          docker run --rm --name benchbase-postgres --network "$docker_network" \
-            --env BENCHBASE_PROFILE=postgres -v /tmp/config.xml:/tmp/config.xml -v "$PWD/results:/benchbase/results" \
-            benchbase:latest -b "$benchmark" -c /tmp/config.xml --create=true --load=true --execute=true
-          ls results/${benchmark}_*.csv
+          for image in benchbase benchbase-postgres; do
+            # Adjust the sample config to talk to the container service instead of localhost.
+            cat "config/postgres/sample_${benchmark}_config.xml" | sed -e 's/localhost:5432/postgres:5432/g' > /tmp/config.xml
+            # Lookup the service container's docker network so we can place the benchbase container in it too.
+            docker_network="$(docker ps --filter expose=5432 --format '{{.Networks}}')"
+            # Map the adjusted config into the container and use it to run the test.
+            rm -rf results
+            mkdir -p results
+            docker run --rm --name benchbase-postgres --network "$docker_network" \
+              --env BENCHBASE_PROFILE=postgres -v /tmp/config.xml:/tmp/config.xml -v "$PWD/results:/benchbase/results" \
+              "$image" -b "$benchmark" -c /tmp/config.xml --create=true --load=true --execute=true
+            # Test that the results files were produced.
+            ls results/${benchmark}_*.csv
+          done
+      # Publish the docker image if the build/test was successful.
+      # Only do this with approved PRs and if the login secrets are available.
+      # Typically we expect to publish to benchbase.azurecr.io,
+      # but setting ACR_LOGINURL to something else allows us to do testing on forks.
+      - name: Log in to Docker Hub
+        if: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) && env.CONTAINER_REGISTRY_NAME != '' }}
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ secrets.ACR_LOGINURL }}
+          username: ${{ secrets.ACR_USERNAME }}
+          password: ${{ secrets.ACR_PASSWORD }}
+      - name: Push Docker image
+        if: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) && env.CONTAINER_REGISTRY_NAME != '' }}
+        run: |
+          docker push -a ${{ secrets.ACR_LOGINURL}}/benchbase-dev
+          docker push -a ${{ secrets.ACR_LOGINURL}}/benchbase
+          for profile in $BENCHBASE_PROFILES; do
+            docker push -a ${{ secrets.ACR_LOGINURL }}/benchbase-$profile
+          done

.github/workflows/maven.yml

@@ -11,6 +11,8 @@ results
 HELP.md
 PRIVATE.md
 target/
+profiles/
+docker/benchbase/fullimage/tmp/
 !.mvn/wrapper/maven-wrapper.jar
 !**/src/main/**
 !**/src/test/**

.gitignore

@@ -170,6 +170,22 @@ To modify the logging level you can update [`logging.properties`](src/main/resou
 
 ### How use with Docker
 
+- Build or pull a dev image to help building from source:
+
+  ```sh
+  ./docker/benchbase/build-dev-image.sh
+  ./docker/benchbase/run-dev-image.sh
+  ```
+
+  or
+
+  ```sh
+  docker run -it --rm --pull \
+    -v /path/to/benchbase-source:/benchbase \
+    -v $HOME/.m2:/home/containeruser/.m2 \
+    benchbase.azure.cr.io/benchbase-dev
+  ```
+
 - Build the full image:
 
   ```sh
@@ -186,7 +202,14 @@ To modify the logging level you can update [`logging.properties`](src/main/resou
   BENCHBASE_PROFILE='postgres' ./docker/benchbase/run-full-image.sh --help # or other benchbase args as before
   ```
 
-> See [scripts](./docker/benchbase/) for further details.
+  or
+
+  ```sh
+  docker run -it --rm --env BENCHBASE_PROFILE='postgres' \
+    -v results:/benchbase/results benchbase.azurecr.io/benchbase --help # or other benchbase args as before
+  ```
+
+> See the [docker/benchbase/README.md](./docker/benchbase/) for further details.
 
 [Github Codespaces](https://github.com/features/codespaces) and [VSCode devcontainer](https://code.visualstudio.com/docs/remote/containers) support is also available.
 

README.md

@@ -0,0 +1,90 @@
+# Benchbase Containers
+
+This directory provides scripts for producing and running two different containers.
+
+## Building Containers
+
+1. `benchbase-dev` which can be used for *building* and *developing* `benchbase`.
+
+    It contains a full JDK and is expected to be run with the local source code checkout mapped into it (e.g. `docker run -v $repo_root:/benchbase benchbase-dev`).
+
+    The [`.devcontainer.json`](../../.devcontainer/devcontainer.json) config references this one and can be used with [Github Codespaces](https://github.com/features/codespaces) or [VS Code dev containers](https://code.visualstudio.com/docs/remote/containers).
+
+    See [`build-dev-image.sh`](./build-dev-image.sh) and [`run-dev-image.sh`](./run-dev-image.sh) for further details.
+
+2. `benchbase` which can be used for easily *running* `benchbase` in a container environment (e.g. via Kubernetes).
+
+    It contains just a JRE and can be used for running prebuilt profiles of `benchbase`.
+
+    The `benchbase-dev` image is used to produce the prebuilt profiles.
+
+    By default all profiles are built, though this can be controlled by setting the `BENCHBASE_PROFILES` environment variable to a specific subset of profiles when building the image with the `build-full-image.sh` script.
+    For instance:
+
+    ```sh
+    BENCHBASE_PROFILES='postgres mysql' ./build-full-image.sh
+    ```
+
+    The test suite can also be skipped during the build phase by setting `SKIP_TESTS='true'`.
+
+    Additionally, an `benchbase-{profile_name}` image with just that profile in it will be created for each of the `BENCHBASE_PROFILES`.
+
+    When running the `benchbase` image, results are placed in `/benchbase/results`, which is expected to be mapped back into the host environment (e.g. `docker run -v /place/to/save/results:/benchbase/results benchbase`)
+
+    See [`build-full-image.sh`](./build-full-image.sh) and [`run-full-image.sh`](./run-full-image.sh) for further details.
+
+### Publishing Containers
+
+These scripts are used to publish the images to a container registry for easy consumption with `docker pull` as well.
+
+If you would like to publish them to an alternative repository, you can setup the appropriate tags during the build by setting the `CONTAINER_REGISTRY_NAME` environment variable.
+For instance:
+
+```sh
+export CONTAINER_REGISTRY_NAME='benchbasedev.azurecr.io'
+./build-dev-image.sh
+./build-full-image.sh
+
+docker push $CONTAINER_REGISTRY_NAME/benchbase-dev
+docker push $CONTAINER_REGISTRY_NAME/benchbase
+```
+
+## Running Containers
+
+### With Local Builds
+
+```sh
+# This will build and run a container shell with maven and java preloaded and the current source checkout mapped into it:
+./docker/benchbase/run-dev-image.sh
+```
+
+
+```sh
+# This will build and run a container for running the postgres benchbase profile:
+BENCHBASE_PROFILE='postgres' ./docker/benchbase/run-full-image.sh
+```
+
+### Prebuilt Containers
+
+To use prebuilt containers, the following can be used:
+
+```sh
+docker pull benchbase.azurecr.io/benchbase-dev
+
+# Provide a build environment for working with the local source code:
+docker run -it --rm -v /path/to/src:/benchbase benchbase.azurecr.io/benchbase-dev
+```
+
+> Optional: also reuse the local `MAVEN_CONFIG` and it's repository download cache with the following argument:
+>
+> `-v "${MAVEN_CONFIG:-$HOME/.m2}:/home/containeruser/.m2"`
+
+```sh
+docker pull benchbase.azurecr.io/benchbase
+
+# Run benchbase against a postgres instance and store the results in /results:
+docker run --rm --env BENCHBASE_PROFILE='postgres' -v /results:/benchbase/results benchbase.azurecr.io/benchbase --help
+
+# Or by referencing the standalone image for that profile:
+docker run --rm -v /results:/benchbase/results benchbase.azurecr.io/benchbase-postgres --help
+```