Merge pull request #11647 from IQSS/develop

Merge the “develop” branch into “master” for 6.7 Release

Author: ofahimIQSS

.github/CODEOWNERS

@@ -0,0 +1,8 @@
+
+# Any container related stuff should be assigned to / reviewed by Oliver and/or Phil
+modules/container-configbaker/** @poikilotherm @pdurbin
+modules/container-base/** @poikilotherm @pdurbin
+src/main/docker/** @poikilotherm @pdurbin
+docker-compose-dev.yml @poikilotherm @pdurbin
+.github/workflows/scripts/containers** @poikilotherm @pdurbin
+.github/workflows/container_* @poikilotherm @pdurbin

.github/workflows/codeql.yml

@@ -0,0 +1,104 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL Advanced"
+
+on:
+  push:
+    branches: [ "develop", "master" ]
+  pull_request:
+    branches: [ "develop", "master" ]
+  schedule:
+    - cron: '30 6 * * 4'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: actions
+          build-mode: none
+        - language: java-kotlin
+          build-mode: none # This mode only analyzes Java. Set this to 'autobuild' or 'manual' to analyze Kotlin too.
+        - language: javascript-typescript
+          build-mode: none
+        - language: python
+          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Add any setup steps before running the `github/codeql-action/init` action.
+    # This includes steps like installing compilers or runtimes (`actions/setup-node`
+    # or others). This is typically only required for manual builds.
+    # - name: Setup runtime (example)
+    #   uses: actions/setup-example@v1
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      shell: bash
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/container_app_pr.yml

@@ -1,15 +1,15 @@
 ---
 name: Preview Application Container Image
 
+# TODO: merge this workflow into the existing container_app_push.yaml flow - there's not much difference!
+
 on:
     # We only run the push commands if we are asked to by an issue comment with the correct command.
     # This workflow is always taken from the default branch and runs in repo context with access to secrets.
     repository_dispatch:
         types: [ push-image-command ]
 
 env:
-    IMAGE_TAG: unstable
-    BASE_IMAGE_TAG: unstable
     PLATFORMS: "linux/amd64,linux/arm64"
 
 jobs:
@@ -65,10 +65,9 @@ jobs:
                   install
             - name: Deploy multi-arch application and configbaker container image
               run: >
-                  mvn 
-                  -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }}
+                  mvn -Pct deploy 
+                  -Dapp.image.tag=${{ env.IMAGE_TAG }}
                   -Ddocker.registry=ghcr.io -Ddocker.platforms=${{ env.PLATFORMS }}
-                  -Pct deploy
 
             - uses: marocchino/sticky-pull-request-comment@v2
               with:

.github/workflows/container_app_push.yml

@@ -24,7 +24,6 @@ env:
     IMAGE_TAG: unstable
     REGISTRY: "" # Empty means default to Docker Hub
     PLATFORMS: "linux/amd64,linux/arm64"
-    MASTER_BRANCH_TAG: alpha
 
 jobs:
     build:
@@ -60,30 +59,6 @@ jobs:
 
             # TODO: add smoke / integration testing here (add "-Pct -DskipIntegrationTests=false")
 
-    hub-description:
-        needs: build
-        name: Push image descriptions to Docker Hub
-        # Run this when triggered via push or schedule as reused workflow from base / maven unit tests.
-        # Excluding PRs here means we will have no trouble with secrets access. Also avoid runs in forks.
-        if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' && github.repository_owner == 'IQSS' }}
-        runs-on: ubuntu-latest
-        steps:
-            - uses: actions/checkout@v4
-            - uses: peter-evans/dockerhub-description@v4
-              with:
-                  username: ${{ secrets.DOCKERHUB_USERNAME }}
-                  password: ${{ secrets.DOCKERHUB_TOKEN }}
-                  repository: gdcc/dataverse
-                  short-description: "Dataverse Application Container Image providing the executable"
-                  readme-filepath: ./src/main/docker/README.md
-            - uses: peter-evans/dockerhub-description@v4
-              with:
-                  username: ${{ secrets.DOCKERHUB_USERNAME }}
-                  password: ${{ secrets.DOCKERHUB_TOKEN }}
-                  repository: gdcc/configbaker
-                  short-description: "Dataverse Config Baker Container Image providing setup tooling and more"
-                  readme-filepath: ./modules/container-configbaker/README.md
-
     # Note: Accessing, pushing tags etc. to DockerHub or GHCR will only succeed in upstream because secrets.
     # We check for them here and subsequent jobs can rely on this to decide if they shall run.
     check-secrets:
@@ -107,13 +82,13 @@ jobs:
         needs: check-secrets
         name: "Package & Publish"
         runs-on: ubuntu-latest
-        # Only run this job if we have access to secrets. This is true for events like push/schedule which run in
+        # Only run this job if we have access to secrets. This is true for events like push/schedule which run in the
         # context of the main repo, but for PRs only true if coming from the main repo! Forks have no secret access.
         #
         # Note: The team's decision was to not auto-deploy an image on any git push where no PR exists (yet).
-        #       Accordingly, only run for push events on branches develop and master.
+        #       Accordingly, only run for push events on the 'develop' branch.
         if: needs.check-secrets.outputs.available == 'true' &&
-            ( github.event_name != 'push' || ( github.event_name == 'push' && contains(fromJSON('["develop", "master"]'), github.ref_name)))
+            ( github.event_name != 'push' || ( github.event_name == 'push' && github.ref_name == 'develop' ))
         steps:
             - name: Checkout and Setup Maven
               uses: IQSS/dataverse/.github/actions/setup-maven@develop
@@ -141,16 +116,15 @@ jobs:
             - name: Set up QEMU for multi-arch builds
               uses: docker/setup-qemu-action@v3
 
-            - name: Re-set image tag based on branch (if master)
-              if: ${{ github.ref_name == 'master' }}
+            - name: Add rolling image tag when pushing to develop
+              if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }}
               run: |
-                  echo "IMAGE_TAG=${{ env.MASTER_BRANCH_TAG }}" >> $GITHUB_ENV
-                  echo "BASE_IMAGE_TAG=${{ env.MASTER_BRANCH_TAG }}" >> $GITHUB_ENV
+                  echo "ADDITIONAL_TAGS=-Ddocker.tags.upcoming=$( mvn initialize help:evaluate -Pct -Dexpression=app.image.tag -Dapp.image.tag='${app.image.version}-${base.image.flavor}' -q -DforceStdout )" | tee -a "$GITHUB_ENV"
             - name: Re-set image tag and container registry when on PR
               if: ${{ github.event_name == 'pull_request' }}
               run: |
-                  echo "IMAGE_TAG=$(echo "$GITHUB_HEAD_REF" | tr '\\/_:&+,;#*' '-')" >> $GITHUB_ENV
-                  echo "REGISTRY='-Ddocker.registry=ghcr.io'" >> $GITHUB_ENV
+                  echo "IMAGE_TAG=$(echo "$GITHUB_HEAD_REF" | tr '\\/_:&+,;#*' '-')" | tee -a "$GITHUB_ENV"
+                  echo "REGISTRY='-Ddocker.registry=ghcr.io'" | tee -a "$GITHUB_ENV"
 
             # Necessary to split as otherwise the submodules are not available (deploy skips install)
             - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests)
@@ -162,7 +136,7 @@ jobs:
             - name: Deploy multi-arch application and configbaker container image
               run: >
                   mvn 
-                  -Dapp.image.tag=${{ env.IMAGE_TAG }}
+                  -Dapp.image.tag=${{ env.IMAGE_TAG }} ${{ env.ADDITIONAL_TAGS }}
                   $( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" )
                   ${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }}
                   -P ct deploy

.github/workflows/container_base_push.yml

@@ -1,10 +1,8 @@
 ---
-name: Container Images Releasing
+name: Base Container Image
 
 on:
     push:
-        tags:
-            - 'v[6-9].**'
         branches:
             - 'develop'
         # "Path filters are not evaluated for pushes of tags" https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore
@@ -33,7 +31,7 @@ jobs:
         # Only run in upstream repo - avoid unnecessary runs in forks
         if: ${{ github.repository_owner == 'IQSS' }}
         outputs:
-          base-image-ref: ${{ steps.finalize.outputs.base-image-ref }}
+          base-image-ref: ${{ steps.determine-name.outputs.full-ref }}
 
         steps:
             - name: Checkout and Setup Maven
@@ -52,12 +50,10 @@ jobs:
             # In case this is a push to develop, we care about buildtime.
             # Configure a remote ARM64 build host in addition to the local AMD64 in two steps.
             - name: Setup SSH agent
-              if: ${{ github.event_name != 'schedule' }}
-              uses: webfactory/ssh-agent@v0.9.0
+              uses: webfactory/ssh-agent@v0.9.1
               with:
                   ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }}
             - name: Provide the known hosts key and the builder config
-              if: ${{ github.event_name != 'schedule' }}
               run: |
                   echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts
                   mkdir -p modules/container-base/target/buildx-state/buildx/instances
@@ -81,40 +77,25 @@ jobs:
 
             # Determine the base image name we are going to use from here on
             - name: Determine base image name
+              id: determine-name
               run: |
-                if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then
-                  echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" | tee -a "${GITHUB_ENV}"
-                  echo "BASE_IMAGE_UPCOMING=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}"
-                else
-                  echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}"
-                fi
-            - name: Calculate revision number for immutable tag (on release branches only)
-              if: ${{ github.ref_name != env.DEVELOPMENT_BRANCH }}
-              id: revision-tag
-              uses: ./.github/actions/get-image-revision
-              with:
-                image-ref: ${{ env.BASE_IMAGE }}
-                tag-options-prefix: "-Dbase.image.tag.suffix='' -Ddocker.tags.revision="
+                BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )
+                BASE_IMAGE_UPCOMING=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )
+                
+                echo "BASE_IMAGE=${BASE_IMAGE}" | tee -a "${GITHUB_ENV}"
+                echo "BASE_IMAGE_UPCOMING=${BASE_IMAGE_UPCOMING}" | tee -a "${GITHUB_ENV}"
+                echo "full-ref=${BASE_IMAGE_UPCOMING}" | tee -a "$GITHUB_OUTPUT"
+
             - name: Configure update of "latest" tag for development branch
               id: develop-tag
-              if: ${{ github.ref_name == env.DEVELOPMENT_BRANCH }}
               run: |
                 echo "tag-options=-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=${BASE_IMAGE_UPCOMING#*:}" | tee -a "${GITHUB_OUTPUT}"
 
             - name: Deploy multi-arch base container image to Docker Hub
               id: build
               run: |
                 mvn -f modules/container-base -Pct deploy -Ddocker.noCache -Ddocker.platforms=${{ env.PLATFORMS }} \
-                  -Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }} ${{ steps.revision-tag.outputs.tag-options }}
-
-            - name: Determine appropriate base image ref for app image
-              id: finalize
-              run: |
-                if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then
-                  echo "base-image-ref=${BASE_IMAGE_UPCOMING}" | tee -a "$GITHUB_OUTPUT"
-                else
-                  echo "base-image-ref=gdcc/base:${{ steps.revision-tag.outputs.revision-tag }}" | tee -a "$GITHUB_OUTPUT"
-                fi
+                  -Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }}
 
     push-app-img:
         name: "Rebase & Publish App Image"