chore(ci): run lint in a separate build before running the tests

.github/dependabot.yml

@@ -4,8 +4,7 @@ version: 2
 updates:
     - package-ecosystem: github-actions
       directories:
-        - / 
-        - /modulegen/_template
+        - /
       schedule:
         interval: monthly
         day: sunday

.github/workflows/ci-lint-go.yml

@@ -0,0 +1,56 @@
+name: Run lint for a Go project
+run-name: "${{ inputs.project-directory }}"
+
+on:
+  workflow_call:
+    inputs:
+      project-directory:
+        required: true
+        type: string
+        default: "."
+        description: "The directory where the Go project is located."
+
+permissions:
+  contents: read
+  # Optional: allow read access to pull request. Use with `only-new-issues` option.
+  # pull-requests: read
+
+jobs:
+  lint-go-project:
+    name: "lint: ${{ inputs.project-directory }}"
+    runs-on: 'ubuntu-latest'
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Set up Go
+        uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
+        with:
+            go-version-file: "${{ inputs.project-directory == '' && '.' || inputs.project-directory }}/go.mod"
+            cache-dependency-path: "${{ inputs.project-directory == '' && '.' || inputs.project-directory }}/go.sum"
+        id: go
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@ec5d18412c0aeab7936cb16880d708ba2a64e1ae # v6.2.0
+        with:
+          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
+          version: v1.63.4
+          # Optional: working directory, useful for monorepos
+          working-directory: ${{ inputs.project-directory }}
+          # Optional: if set to true then the all caching functionality will be complete disabled,
+          #           takes precedence over all other caching options.
+          skip-cache: true
+
+      - name: generate
+        working-directory: ./${{ inputs.project-directory }}
+        shell: bash
+        run: |
+          make generate
+          git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]]
+
+      - name: modTidy
+        working-directory: ./${{ inputs.project-directory }}
+        shell: bash
+        run: |
+          make tidy
+          git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]]

.github/workflows/ci-test-go.yml

@@ -1,5 +1,5 @@
 name: Run tests for a Go project
-run-name: "${{ inputs.project-directory }} ${{ inputs.go-version }} ${{ inputs.platform }}"
+run-name: "${{ inputs.project-directory }} ${{ inputs.go-version }} ${{ inputs.platforms }}"
 
 on:
   workflow_call:
@@ -8,15 +8,11 @@ on:
         required: true
         type: string
         description: "The version of Go to use for the test."
-      platform:
+      platforms:
         required: true
         type: string
-        description: "The platform to run the test on."
-      fail-fast:
-        required: false
-        type: boolean
-        default: true
-        description: "Fail the workflow if any of the jobs fail."
+        default: "ubuntu-latest"
+        description: "The platforms in which the project will be run"
       project-directory:
         required: true
         type: string
@@ -27,11 +23,6 @@ on:
         type: boolean
         default: false
         description: "Run the test with rootless docker."
-      run-tests:
-        required: false
-        type: boolean
-        default: true
-        description: "Run the tests under conditions controlled by the caller workflow."
       ryuk-disabled:
         required: false
         type: boolean
@@ -45,13 +36,16 @@ permissions:
 
 jobs:
   test-go-project:
-    name: "${{ inputs.project-directory }}/${{ inputs.platform }}/${{ inputs.go-version }}"
-    runs-on: ${{ inputs.platform }}
-    continue-on-error: ${{ !inputs.fail-fast }}
+    name: "test: ${{ inputs.project-directory }}/${{ inputs.go-version }}"
+    # Modulegen can run the tests on all platforms
     env:
       TESTCONTAINERS_RYUK_DISABLED: "${{ inputs.ryuk-disabled }}"
       RYUK_CONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '5m' || '60s' }}"
       RYUK_RECONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '30s' || '10s' }}"
+    strategy:
+      matrix:
+        platform: ${{ fromJSON(inputs.platforms) }}
+    runs-on: ${{ matrix.platform }}
     steps:
       - name: Setup rootless Docker
         if: ${{ inputs.rootless-docker }}
@@ -60,49 +54,15 @@ jobs:
           rootless: true
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Set up Go
-        uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5
+        uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
         with:
             go-version: '${{ inputs.go-version }}'
             cache-dependency-path: '${{ inputs.project-directory }}/go.sum'
         id: go
 
-      - name: golangci-lint
-        if: ${{ inputs.platform == 'ubuntu-latest' }}
-        uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1
-        with:
-          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
-          version: v1.61.0
-          # Optional: working directory, useful for monorepos
-          working-directory: ${{ inputs.project-directory }}
-          # Optional: golangci-lint command line arguments.
-          args: --verbose
-          # Optional: if set to true then the all caching functionality will be complete disabled,
-          #           takes precedence over all other caching options.
-          skip-cache: true
-
-      - name: generate
-        if: ${{ inputs.platform == 'ubuntu-latest' }}
-        working-directory: ./${{ inputs.project-directory }}
-        shell: bash
-        run: |
-          make generate
-          git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]]
-
-      - name: modVerify
-        working-directory: ./${{ inputs.project-directory }}
-        run: go mod verify
-
-      - name: modTidy
-        if: ${{ inputs.platform == 'ubuntu-latest' }}
-        working-directory: ./${{ inputs.project-directory }}
-        shell: bash
-        run: |
-          make tidy
-          git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]]
-
       - name: ensure compilation
         working-directory: ./${{ inputs.project-directory }}
         run: go build
@@ -118,11 +78,6 @@ jobs:
           fi
 
       - name: go test
-        # only run tests on linux, there are a number of things that won't allow the tests to run on anything else
-        # many (maybe, all?) images used can only be build on Linux, they don't have Windows in their manifest, and
-        # we can't put Windows Server in "Linux Mode" in GitHub actions
-        # another, host mode is only available on Linux, and we have tests around that, do we skip them?
-        if: ${{ inputs.run-tests }}
         working-directory: ./${{ inputs.project-directory }}
         timeout-minutes: 30
         run: make test-unit
@@ -135,7 +90,7 @@ jobs:
           echo "ARTIFACT_NAME=$(basename ${{ inputs.project-directory == '.' && 'core' || inputs.project-directory }})-${{ inputs.go-version }}-${{ inputs.platform }}" >> $GITHUB_ENV
 
       - name: Upload SonarCloud files
-        if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && inputs.platform == 'ubuntu-latest' && inputs.run-tests && !inputs.rootless-docker && !inputs.ryuk-disabled }}
+        if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && matrix.platform == 'ubuntu-latest' && !inputs.rootless-docker && !inputs.ryuk-disabled }}
         uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
           name: sonarcloud-${{ env.ARTIFACT_NAME }}

.github/workflows/ci.yml

@@ -1,131 +1,124 @@
-# This file is autogenerated by the 'modulegen' tool.
-# Please update the 'ci.yml' template instead.
 name: Main pipeline
 
 on:
   push:
     branches:
       - main
-    paths-ignore:
-      - '.vscode/**'
-      - 'mkdocs.yml'
-      - 'docs/**'
-      - 'README.md'
   pull_request:
-    paths-ignore:
-      - '.vscode/**'
-      - 'mkdocs.yml'
-      - 'docs/**'
-      - 'README.md'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.sha }}
   cancel-in-progress: true
 
 jobs:
+  detect-modules:
+    runs-on: ubuntu-latest
+    outputs:
+      modules: ${{ steps.set-modified-modules.outputs.modules }}
+      modules_count: ${{ steps.set-modified-modules-count.outputs.modules_count }}
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - id: changed-files
+        name: Get changed files
+        uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45.0.4
+
+      - id: set-modified-modules
+        name: Set all modified modules
+        env:
+          ALL_CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}"
+        run: echo "modules=$(./scripts/changed-modules.sh)" >> $GITHUB_OUTPUT
+
+      - id: set-modified-modules-count
+        name: Set all modified modules count
+        run: echo "modules_count=$(echo ${{ toJSON(steps.set-modified-modules.outputs.modules) }} | jq '. | length')" >> $GITHUB_OUTPUT
+
+      - name: Print out the modules to be used
+        run: |
+          echo "${{ steps.set-modified-modules-count.outputs.modules_count }} modules in the build"
+          echo "${{ steps.set-modified-modules.outputs.modules }}"
+
+  lint:
+    # only run if there are modules to lint
+    if: ${{ needs.detect-modules.outputs.modules_count > 0  }}
+    needs:
+      - detect-modules
+    strategy:
+      matrix:
+        module: ${{ fromJSON(needs.detect-modules.outputs.modules) }}
+    uses: ./.github/workflows/ci-lint-go.yml
+    with:
+      project-directory: "${{ matrix.module }}"
+
   test:
+    # only run if there are modules to test
+    if: ${{ needs.detect-modules.outputs.modules_count > 0  }}
+    needs:
+      - detect-modules
+      - lint
     strategy:
+      # We don't want to fail the build the soonest but identify which modules passed and failed.
+      fail-fast: false
       matrix:
         go-version: [1.22.x, 1.x]
-        platform: [ubuntu-latest, macos-latest]
+        module: ${{ fromJSON(needs.detect-modules.outputs.modules) }}
     uses: ./.github/workflows/ci-test-go.yml
     with:
       go-version: ${{ matrix.go-version }}
-      fail-fast: true
-      platform: ${{ matrix.platform }}
-      project-directory: "."
+      platforms: ${{ matrix.module == 'modulegen' && '["ubuntu-latest", "macos-latest", "windows-latest"]' || '["ubuntu-latest"]' }}
+      project-directory: "${{ matrix.module }}"
       rootless-docker: false
-      run-tests: ${{ matrix.platform == 'ubuntu-latest' }}
       ryuk-disabled: false
 
   # The job below is a copy of the job above, but with ryuk disabled.
   # It's executed in the first stage to avoid concurrency issues.
   test-reaper-off:
+    # the core module is identified by the empty string (the root path)
+    if: ${{ contains(fromJSON(needs.detect-modules.outputs.modules), '') }}
+    needs:
+      - detect-modules
+      - lint
     name: "Test with reaper off"
     strategy:
       matrix:
         go-version: [1.22.x, 1.x]
     uses: ./.github/workflows/ci-test-go.yml
     with:
       go-version: ${{ matrix.go-version }}
-      fail-fast: false
-      platform: "ubuntu-latest"
+      platforms: '["ubuntu-latest"]'
       project-directory: "."
       rootless-docker: false
-      run-tests: true
       ryuk-disabled: true
 
   # The job below is a copy of the job above, but with Docker rootless.
   # It's executed in the first stage to avoid concurrency issues.
   test-rootless-docker:
+    # the core module is identified by the empty string (the root path)
+    if: ${{ contains(fromJSON(needs.detect-modules.outputs.modules), '') }}
+    needs:
+      - detect-modules
+      - lint
     name: "Test with Rootless Docker"
     strategy:
       matrix:
         go-version: [1.22.x, 1.x]
-        platform: [ubuntu-latest]
     uses: ./.github/workflows/ci-test-go.yml
     with:
       go-version: ${{ matrix.go-version }}
-      fail-fast: false
-      platform: "ubuntu-latest"
+      platforms: '["ubuntu-latest"]'
       project-directory: "."
       rootless-docker: true
-      run-tests: true
-      ryuk-disabled: false
-
-  test-module-generator:
-    strategy:
-      matrix:
-        go-version: [1.22.x, 1.x]
-        platform: [ubuntu-latest, macos-latest, windows-latest]
-    uses: ./.github/workflows/ci-test-go.yml
-    with:
-      go-version: ${{ matrix.go-version }}
-      fail-fast: true
-      platform: ${{ matrix.platform }}
-      project-directory: "modulegen"
-      rootless-docker: false
-      run-tests: true
-      ryuk-disabled: false
-
-  test-modules:
-    needs: test
-    strategy:
-      matrix:
-        go-version: [1.22.x, 1.x]
-        platform: [ubuntu-latest]
-        module: [artemis, azurite, cassandra, chroma, clickhouse, cockroachdb, compose, consul, couchbase, databend, dolt, dynamodb, elasticsearch, etcd, gcloud, grafana-lgtm, inbucket, influxdb, k3s, k6, kafka, localstack, mariadb, meilisearch, milvus, minio, mockserver, mongodb, mssql, mysql, nats, neo4j, ollama, openfga, openldap, opensearch, pinecone, postgres, pulsar, qdrant, rabbitmq, redis, redpanda, registry, surrealdb, valkey, vault, vearch, weaviate, yugabytedb]
-    uses: ./.github/workflows/ci-test-go.yml
-    with:
-      go-version: ${{ matrix.go-version }}
-      fail-fast: false
-      platform: ${{ matrix.platform }}
-      project-directory: modules/${{ matrix.module }}
-      rootless-docker: false
-      run-tests: ${{ matrix.platform == 'ubuntu-latest' }}
-      ryuk-disabled: false
-
-  test-examples:
-    needs: test-modules
-    strategy:
-      matrix:
-        module: [nginx, toxiproxy]
-    uses: ./.github/workflows/ci-test-go.yml
-    with:
-      go-version: "1.22.x"
-      fail-fast: true
-      platform: 'ubuntu-latest'
-      project-directory: examples/${{ matrix.module }}
-      rootless-docker: false
-      run-tests: true
       ryuk-disabled: false
 
   sonarcloud:
     permissions:
       contents: read  # for actions/checkout to fetch code
       pull-requests: read  # for sonarsource/sonarcloud-github-action to determine which PR to decorate
-    if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' }}
-    needs: test-examples
+    if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && needs.detect-modules.outputs.modules_count > 0 }}
+    needs:
+      - detect-modules
+      - test
     runs-on: ubuntu-latest 
     steps:
       - name: Check out code into the Go module directory