Dev build cycle improvements (#277)

- Adds more assembly descriptors for producing flat directories to avoid having to compress/extract when building containers.
- Provides a way to selectively enable that during container builds.
- Skips generating the zip files during the CI pipelines (they were unused anyways).
- Adds some unrelated minor fixups to more docker-compose scripts following #276

Author: bpkroth

.github/workflows/maven.yml

@@ -70,7 +70,7 @@ jobs:
         distribution: 'temurin'
 
     - name: Package with Maven
-      run: mvn -B package -P ${{matrix.profile}} --file pom.xml -DskipTests
+      run: mvn -B package -P ${{matrix.profile}} --file pom.xml -DskipTests -D descriptors=src/main/assembly/tgz.xml
 
     - name: Upload TGZ artifact
       uses: actions/upload-artifact@v3

docker/benchbase/common-env.sh

@@ -75,7 +75,7 @@ if ! docker buildx version >/dev/null 2>&1; then
     echo 'NOTE: docker buildkit is unavailable.' >&2
     DOCKER_BUILDKIT=0
     docker_build_args=''
-elif [ -z "$DOCKER_BUILDKIT" ]; then
+elif [ -z "${DOCKER_BUILDKIT:-}" ]; then
     # If not already set, default to buildkit.
     DOCKER_BUILDKIT=1
 fi

docker/benchbase/devcontainer/build-in-container.sh

@@ -11,16 +11,27 @@ SKIP_TESTS="${SKIP_TESTS:-false}"
 
 cd /benchbase
 mkdir -p results
+mkdir -p profiles
+
+SKIP_TEST_ARGS='-D skipTests -D maven.test.skip -D maven.javadoc.skip=true'
+EXTRA_MAVEN_ARGS="${EXTRA_MAVEN_ARGS:-}"
+
+if [ "$CLEAN_BUILD" == false ]; then
+    # In tight dev build loops we want to avoid regenerating classes when only
+    # the git properties have changed.
+    EXTRA_MAVEN_ARGS+=" -D maven.gitcommitid.skip=true"
+fi
 
 function build_profile() {
     local profile="$1"
     # Build in a separate directory so we can do it in parallel.
     rm -rf profiles/$profile/
     mkdir -p target/$profile
-    mvn -T 2C -B --file pom.xml package -P $profile -D skipTests -D buildDirectory=target/$profile
-    # Extract the resultant package.
-    mkdir -p profiles/$profile
-    tar -C profiles/$profile/ --strip-components=1 -xvzf target/$profile/benchbase-$profile.tgz
+    # Build the profile without tests (we did that separately).
+    mvn -T 2C -B --file pom.xml package -D descriptors=src/main/assembly/dir.xml -P $profile \
+        $SKIP_TEST_ARGS $EXTRA_MAVEN_ARGS -D buildDirectory=target/$profile
+    # Copy the resultant output to the profiles directory.
+    cp -rlv target/$profile/benchbase-$profile/benchbase-$profile profiles/$profile
     # Later the container entrypoint will move into this directory to run it, so
     # save all of the results back to the common volume mapping location.
     ln -s ../../results profiles/$profile/results
@@ -62,7 +73,7 @@ function clean_profile_build() {
     local profile="$1"
     echo "INFO: Cleaning profile $profile"
     if [ -d target/$profile ]; then
-        mvn -B --file pom.xml -D buildDirectory=target/$profile clean
+        mvn -B --file pom.xml $SKIP_TEST_ARGS $EXTRA_MAVEN_ARGS -D buildDirectory=target/$profile clean
         rm -rf target/$profile
     fi
 }
@@ -85,14 +96,14 @@ fi
 # Fetch resources serially to work around mvn races with downloading the same
 # file in multiple processes (mvn uses *.part instead of use tmpfile naming).
 for profile in ${BENCHBASE_PROFILES}; do
-    mvn -T2C -B --file pom.xml -D buildDirectory=target/$profile process-resources dependency:copy-dependencies
+    mvn -T2C -B --file pom.xml -D buildDirectory=target/$profile $EXTRA_MAVEN_ARGS process-resources dependency:copy-dependencies
 done
 
 # Make sure that we've built the base stuff (and test) before we build individual profiles.
-mvn -T 2C -B --file pom.xml compile # ${TEST_TARGET:-}
+mvn -T 2C -B --file pom.xml $SKIP_TEST_ARGS $EXTRA_MAVEN_ARGS compile # ${TEST_TARGET:-}
 if [ -n "${TEST_TARGET:-}" ]; then
     # FIXME: Run tests without parallelism to work around some buggy behavior.
-    mvn -B --file pom.xml $TEST_TARGET
+    mvn -B --file pom.xml $EXTRA_MAVEN_ARGS $TEST_TARGET
 fi
 
 for profile in ${BENCHBASE_PROFILES}; do

docker/benchbase/fullimage/Dockerfile

@@ -22,7 +22,7 @@ VOLUME /benchbase/results
 # Only copy the content necessary for running (not building) benchbase over to the image.
 # These files should have been built using the devcontainer by the
 # build-full-image.sh script.
-COPY . /benchbase/
+COPY --chown=containeruser:containergroup . /benchbase/
 
 ENTRYPOINT ["/benchbase/entrypoint.sh"]
 CMD ["--help"]

docker/benchbase/run-dev-image.sh

@@ -36,6 +36,7 @@ docker run ${INTERACTIVE_ARGS:-} --rm \
     --env BENCHBASE_PROFILES="$BENCHBASE_PROFILES" \
     --env CLEAN_BUILD="$CLEAN_BUILD" \
     --env SKIP_TESTS="$SKIP_TESTS" \
+    --env EXTRA_MAVEN_ARGS="${EXTRA_MAVEN_ARGS:-}" \
     --user "$CONTAINERUSER_UID:$CONTAINERUSER_GID" \
     -v "$MAVEN_CONFIG:/home/containeruser/.m2" \
     -v "$SRC_DIR:/benchbase" benchbase-dev:latest $*

docker/benchbase/run-full-image.sh

@@ -15,9 +15,9 @@ if ! docker image ls --quiet benchbase-$BENCHBASE_PROFILE:latest | grep -q .; th
     CLEAN_BUILD=true
 fi
 
-if [ "$CLEAN_BUILD" != 'false' ]; then
+#if [ "$CLEAN_BUILD" != 'false' ]; then
     ./build-full-image.sh
-fi
+#fi
 
 if [ "$imagename" != 'benchbase' ]; then
     echo "ERROR: Unexpected imagename: $imagename" >&2
@@ -32,7 +32,7 @@ cd "$rootdir"
 mkdir -p results/
 set -x
 docker run -it --rm \
-    "${EXTRA_DOCKER_ARGS:-}" \
+    ${EXTRA_DOCKER_ARGS:-} \
     --env=http_proxy="${http_proxy:-}" --env=https_proxy="${https_proxy:-}" \
     --env BENCHBASE_PROFILE="$BENCHBASE_PROFILE" \
     --user "$CONTAINERUSER_UID:$CONTAINERUSER_GID" \

docker/build-run-benchmark-with-docker.sh

@@ -17,15 +17,30 @@ scriptdir=$(dirname "$(readlink -f "$0")")
 rootdir=$(readlink -f "$scriptdir/..")
 cd "$rootdir"
 
-if [ ! -x "docker/${BENCHBASE_PROFILE}-latest/up.sh" ]; then
-    echo "ERROR: No docker up.sh script available for '$BENCHBASE_PROFILE'"
+if [ "$BENCHBASE_PROFILE" == 'sqlite' ]; then
+    # Map the sqlite db back to the host.
+    touch $PWD/$benchmark.db
+    SRC_DIR="$PWD"
+    if [ -n "$LOCAL_WORKSPACE_FOLDER" ]; then
+        SRC_DIR="$LOCAL_WORKSPACE_FOLDER"
+    fi
+    EXTRA_DOCKER_ARGS="-v $SRC_DIR/$benchmark.db:/benchbase/profiles/sqlite/$benchmark.db"
+else
+    if [ ! -x "docker/${BENCHBASE_PROFILE}-latest/up.sh" ]; then
+        echo "ERROR: No docker up.sh script available for '$BENCHBASE_PROFILE'"
+    fi
+
+    "./docker/${BENCHBASE_PROFILE}-latest/up.sh"
 fi
 
-"./docker/${BENCHBASE_PROFILE}-latest/up.sh"
+CREATE_DB_ARGS='--create=true --load=true'
+if [ "${SKIP_LOAD_DB:-false}" == 'true' ]; then
+    CREATE_DB_ARGS=''
+fi
 
-SKIP_TESTS=${SKIP_TESTS:-true} EXTRA_DOCKER_ARGS="--network=host" \
+SKIP_TESTS=${SKIP_TESTS:-true} EXTRA_DOCKER_ARGS="--network=host $EXTRA_DOCKER_ARGS" \
 ./docker/benchbase/run-full-image.sh \
     --config "config/sample_${benchmark}_config.xml" --bench "$benchmark" \
-    --create=true --load=true --execute=true \
+    $CREATE_DB_ARGS --execute=true \
     --sample 1 --interval-monitor 1000 \
     --json-histograms results/histograms.json

docker/cockroach-latest/prune.sh

@@ -1,3 +1,7 @@
 #!/bin/bash
 
-docker system prune -a -f --volumes
+set -eu
+scriptdir=$(dirname "$(readlink -f "$0")")
+cd "$scriptdir/"
+
+docker system prune -a -f --volumes

docker/mariadb-latest/down.sh

@@ -1,3 +1,7 @@
 #!/bin/bash
 
-docker system prune -a -f --volumes
+set -eu
+scriptdir=$(dirname "$(readlink -f "$0")")
+cd "$scriptdir/"
+
+docker system prune -a -f --volumes