diff --git a/.github/dependabot.yml b/.github/dependabot.yml index cbfa40e5f4..6ce9ed6c57 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,8 +4,7 @@ version: 2 updates: - package-ecosystem: github-actions directories: - - / - - /modulegen/_template + - / schedule: interval: monthly day: sunday diff --git a/.github/workflows/ci-lint-go.yml b/.github/workflows/ci-lint-go.yml new file mode 100644 index 0000000000..2aebbd74c1 --- /dev/null +++ b/.github/workflows/ci-lint-go.yml @@ -0,0 +1,53 @@ +name: Run lint for a Go project +run-name: "${{ inputs.project-directory }}" + +on: + workflow_call: + inputs: + project-directory: + required: true + type: string + default: "." + description: "The directory where the Go project is located." + +permissions: + contents: read + # Optional: allow read access to pull request. Use with `only-new-issues` option. + # pull-requests: read + +jobs: + lint-go-project: + name: "lint: ${{ inputs.project-directory }}" + runs-on: 'ubuntu-latest' + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Set up Go + uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 + with: + go-version-file: "${{ inputs.project-directory == '' && '.' || inputs.project-directory }}/go.mod" + cache-dependency-path: "${{ inputs.project-directory == '' && '.' || inputs.project-directory }}/go.sum" + id: go + + - name: golangci-lint + uses: golangci/golangci-lint-action@ec5d18412c0aeab7936cb16880d708ba2a64e1ae # v6.2.0 + with: + # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version + version: v1.63.4 + # Optional: working directory, useful for monorepos + working-directory: ${{ inputs.project-directory }} + + - name: generate + working-directory: ./${{ inputs.project-directory }} + shell: bash + run: | + make generate + git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] + + - name: modTidy + working-directory: ./${{ inputs.project-directory }} + shell: bash + run: | + make tidy + git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index 38838e0c7b..e9336beb86 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -1,5 +1,5 @@ name: Run tests for a Go project -run-name: "${{ inputs.project-directory }} ${{ inputs.go-version }} ${{ inputs.platform }}" +run-name: "${{ inputs.project-directory }} ${{ inputs.go-version }} ${{ inputs.platforms }}" on: workflow_call: @@ -8,15 +8,11 @@ on: required: true type: string description: "The version of Go to use for the test." - platform: + platforms: required: true type: string - description: "The platform to run the test on." - fail-fast: - required: false - type: boolean - default: true - description: "Fail the workflow if any of the jobs fail." + default: "ubuntu-latest" + description: "The platforms in which the project will be run" project-directory: required: true type: string @@ -27,11 +23,6 @@ on: type: boolean default: false description: "Run the test with rootless docker." - run-tests: - required: false - type: boolean - default: true - description: "Run the tests under conditions controlled by the caller workflow." ryuk-disabled: required: false type: boolean @@ -45,13 +36,16 @@ permissions: jobs: test-go-project: - name: "${{ inputs.project-directory }}/${{ inputs.platform }}/${{ inputs.go-version }}" - runs-on: ${{ inputs.platform }} - continue-on-error: ${{ !inputs.fail-fast }} + name: "test: ${{ inputs.project-directory }}/${{ inputs.go-version }}" + # Modulegen can run the tests on all platforms env: TESTCONTAINERS_RYUK_DISABLED: "${{ inputs.ryuk-disabled }}" RYUK_CONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '5m' || '60s' }}" RYUK_RECONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '30s' || '10s' }}" + strategy: + matrix: + platform: ${{ fromJSON(inputs.platforms) }} + runs-on: ${{ matrix.platform }} steps: - name: Setup rootless Docker if: ${{ inputs.rootless-docker }} @@ -60,49 +54,15 @@ jobs: rootless: true - name: Check out code into the Go module directory - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Set up Go - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5 + uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 with: go-version: '${{ inputs.go-version }}' cache-dependency-path: '${{ inputs.project-directory }}/go.sum' id: go - - name: golangci-lint - if: ${{ inputs.platform == 'ubuntu-latest' }} - uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 - with: - # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version - version: v1.61.0 - # Optional: working directory, useful for monorepos - working-directory: ${{ inputs.project-directory }} - # Optional: golangci-lint command line arguments. - args: --verbose - # Optional: if set to true then the all caching functionality will be complete disabled, - # takes precedence over all other caching options. - skip-cache: true - - - name: generate - if: ${{ inputs.platform == 'ubuntu-latest' }} - working-directory: ./${{ inputs.project-directory }} - shell: bash - run: | - make generate - git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] - - - name: modVerify - working-directory: ./${{ inputs.project-directory }} - run: go mod verify - - - name: modTidy - if: ${{ inputs.platform == 'ubuntu-latest' }} - working-directory: ./${{ inputs.project-directory }} - shell: bash - run: | - make tidy - git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] - - name: ensure compilation working-directory: ./${{ inputs.project-directory }} run: go build @@ -118,11 +78,6 @@ jobs: fi - name: go test - # only run tests on linux, there are a number of things that won't allow the tests to run on anything else - # many (maybe, all?) images used can only be build on Linux, they don't have Windows in their manifest, and - # we can't put Windows Server in "Linux Mode" in GitHub actions - # another, host mode is only available on Linux, and we have tests around that, do we skip them? - if: ${{ inputs.run-tests }} working-directory: ./${{ inputs.project-directory }} timeout-minutes: 30 run: make test-unit @@ -135,7 +90,7 @@ jobs: echo "ARTIFACT_NAME=$(basename ${{ inputs.project-directory == '.' && 'core' || inputs.project-directory }})-${{ inputs.go-version }}-${{ inputs.platform }}" >> $GITHUB_ENV - name: Upload SonarCloud files - if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && inputs.platform == 'ubuntu-latest' && inputs.run-tests && !inputs.rootless-docker && !inputs.ryuk-disabled }} + if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && matrix.platform == 'ubuntu-latest' && !inputs.rootless-docker && !inputs.ryuk-disabled }} uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 with: name: sonarcloud-${{ env.ARTIFACT_NAME }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96382b1ec4..6749573092 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,46 +1,84 @@ -# This file is autogenerated by the 'modulegen' tool. -# Please update the 'ci.yml' template instead. name: Main pipeline on: push: branches: - main - paths-ignore: - - '.vscode/**' - - 'mkdocs.yml' - - 'docs/**' - - 'README.md' pull_request: - paths-ignore: - - '.vscode/**' - - 'mkdocs.yml' - - 'docs/**' - - 'README.md' concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.sha }} cancel-in-progress: true jobs: + detect-modules: + runs-on: ubuntu-latest + outputs: + modules: ${{ steps.set-modified-modules.outputs.modules }} + modules_count: ${{ steps.set-modified-modules-count.outputs.modules_count }} + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - id: changed-files + name: Get changed files + uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45.0.4 + + - id: set-modified-modules + name: Set all modified modules + env: + ALL_CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}" + run: echo "modules=$(./scripts/changed-modules.sh)" >> $GITHUB_OUTPUT + + - id: set-modified-modules-count + name: Set all modified modules count + run: echo "modules_count=$(echo ${{ toJSON(steps.set-modified-modules.outputs.modules) }} | jq '. | length')" >> $GITHUB_OUTPUT + + - name: Print out the modules to be used + run: | + echo "${{ steps.set-modified-modules-count.outputs.modules_count }} modules in the build" + echo "${{ steps.set-modified-modules.outputs.modules }}" + + lint: + # only run if there are modules to lint + if: ${{ needs.detect-modules.outputs.modules_count > 0 }} + needs: + - detect-modules + strategy: + matrix: + module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} + uses: ./.github/workflows/ci-lint-go.yml + with: + project-directory: "${{ matrix.module }}" + test: + # only run if there are modules to test + if: ${{ needs.detect-modules.outputs.modules_count > 0 }} + needs: + - detect-modules + - lint strategy: + # We don't want to fail the build the soonest but identify which modules passed and failed. + fail-fast: false matrix: go-version: [1.22.x, 1.x] - platform: [ubuntu-latest, macos-latest] + module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} uses: ./.github/workflows/ci-test-go.yml with: go-version: ${{ matrix.go-version }} - fail-fast: true - platform: ${{ matrix.platform }} - project-directory: "." + platforms: ${{ matrix.module == 'modulegen' && '["ubuntu-latest", "macos-latest", "windows-latest"]' || '["ubuntu-latest"]' }} + project-directory: "${{ matrix.module }}" rootless-docker: false - run-tests: ${{ matrix.platform == 'ubuntu-latest' }} ryuk-disabled: false # The job below is a copy of the job above, but with ryuk disabled. # It's executed in the first stage to avoid concurrency issues. test-reaper-off: + # the core module is identified by the empty string (the root path) + if: ${{ contains(fromJSON(needs.detect-modules.outputs.modules), '') }} + needs: + - detect-modules + - lint name: "Test with reaper off" strategy: matrix: @@ -48,84 +86,39 @@ jobs: uses: ./.github/workflows/ci-test-go.yml with: go-version: ${{ matrix.go-version }} - fail-fast: false - platform: "ubuntu-latest" + platforms: '["ubuntu-latest"]' project-directory: "." rootless-docker: false - run-tests: true ryuk-disabled: true # The job below is a copy of the job above, but with Docker rootless. # It's executed in the first stage to avoid concurrency issues. test-rootless-docker: + # the core module is identified by the empty string (the root path) + if: ${{ contains(fromJSON(needs.detect-modules.outputs.modules), '') }} + needs: + - detect-modules + - lint name: "Test with Rootless Docker" strategy: matrix: go-version: [1.22.x, 1.x] - platform: [ubuntu-latest] uses: ./.github/workflows/ci-test-go.yml with: go-version: ${{ matrix.go-version }} - fail-fast: false - platform: "ubuntu-latest" + platforms: '["ubuntu-latest"]' project-directory: "." rootless-docker: true - run-tests: true - ryuk-disabled: false - - test-module-generator: - strategy: - matrix: - go-version: [1.22.x, 1.x] - platform: [ubuntu-latest, macos-latest, windows-latest] - uses: ./.github/workflows/ci-test-go.yml - with: - go-version: ${{ matrix.go-version }} - fail-fast: true - platform: ${{ matrix.platform }} - project-directory: "modulegen" - rootless-docker: false - run-tests: true - ryuk-disabled: false - - test-modules: - needs: test - strategy: - matrix: - go-version: [1.22.x, 1.x] - platform: [ubuntu-latest] - module: [artemis, azurite, cassandra, chroma, clickhouse, cockroachdb, compose, consul, couchbase, databend, dolt, dynamodb, elasticsearch, etcd, gcloud, grafana-lgtm, inbucket, influxdb, k3s, k6, kafka, localstack, mariadb, meilisearch, milvus, minio, mockserver, mongodb, mssql, mysql, nats, neo4j, ollama, openfga, openldap, opensearch, pinecone, postgres, pulsar, qdrant, rabbitmq, redis, redpanda, registry, surrealdb, valkey, vault, vearch, weaviate, yugabytedb] - uses: ./.github/workflows/ci-test-go.yml - with: - go-version: ${{ matrix.go-version }} - fail-fast: false - platform: ${{ matrix.platform }} - project-directory: modules/${{ matrix.module }} - rootless-docker: false - run-tests: ${{ matrix.platform == 'ubuntu-latest' }} - ryuk-disabled: false - - test-examples: - needs: test-modules - strategy: - matrix: - module: [nginx, toxiproxy] - uses: ./.github/workflows/ci-test-go.yml - with: - go-version: "1.22.x" - fail-fast: true - platform: 'ubuntu-latest' - project-directory: examples/${{ matrix.module }} - rootless-docker: false - run-tests: true ryuk-disabled: false sonarcloud: permissions: contents: read # for actions/checkout to fetch code pull-requests: read # for sonarsource/sonarcloud-github-action to determine which PR to decorate - if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' }} - needs: test-examples + if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && needs.detect-modules.outputs.modules_count > 0 }} + needs: + - detect-modules + - test runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 2c899be9d4..68d0eebc1e 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -49,7 +49,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/docker-moby-latest.yml b/.github/workflows/docker-moby-latest.yml index 8de58b7c57..db7b837117 100644 --- a/.github/workflows/docker-moby-latest.yml +++ b/.github/workflows/docker-moby-latest.yml @@ -22,18 +22,15 @@ jobs: echo "containerd_integration=${{ matrix.containerd-integration == true && 'containerd' || '' }}" >> "$GITHUB_ENV" - name: Check out code into the Go module directory - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Set up Go - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5 + uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 with: go-version-file: 'go.mod' cache-dependency-path: 'go.sum' id: go - - name: modVerify - run: go mod verify - - name: modTidy run: go mod tidy diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 134ec1ceaf..b804676cf9 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -20,7 +20,7 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: persist-credentials: false diff --git a/modulegen/_template/ci.yml.tmpl b/modulegen/_template/ci.yml.tmpl deleted file mode 100644 index 44872202a4..0000000000 --- a/modulegen/_template/ci.yml.tmpl +++ /dev/null @@ -1,145 +0,0 @@ -# This file is autogenerated by the 'modulegen' tool. -# Please update the 'ci.yml' template instead. -name: Main pipeline - -on: - push: - branches: - - main - paths-ignore: - - '.vscode/**' - - 'mkdocs.yml' - - 'docs/**' - - 'README.md' - pull_request: - paths-ignore: - - '.vscode/**' - - 'mkdocs.yml' - - 'docs/**' - - 'README.md' - -concurrency: - group: {{ "${{ github.workflow }}-${{ github.head_ref || github.sha }}" }} - cancel-in-progress: true - -jobs: - test: - strategy: - matrix: - go-version: [1.22.x, 1.x] - platform: [ubuntu-latest, macos-latest] - uses: ./.github/workflows/ci-test-go.yml - with: - go-version: {{ "${{ matrix.go-version }}" }} - fail-fast: true - platform: {{ "${{ matrix.platform }}" }} - project-directory: "." - rootless-docker: false - run-tests: {{ "${{ matrix.platform == 'ubuntu-latest' }}" }} - ryuk-disabled: false - - # The job below is a copy of the job above, but with ryuk disabled. - # It's executed in the first stage to avoid concurrency issues. - test-reaper-off: - name: "Test with reaper off" - strategy: - matrix: - go-version: [1.22.x, 1.x] - uses: ./.github/workflows/ci-test-go.yml - with: - go-version: {{ "${{ matrix.go-version }}" }} - fail-fast: false - platform: "ubuntu-latest" - project-directory: "." - rootless-docker: false - run-tests: true - ryuk-disabled: true - - # The job below is a copy of the job above, but with Docker rootless. - # It's executed in the first stage to avoid concurrency issues. - test-rootless-docker: - name: "Test with Rootless Docker" - strategy: - matrix: - go-version: [1.22.x, 1.x] - platform: [ubuntu-latest] - uses: ./.github/workflows/ci-test-go.yml - with: - go-version: {{ "${{ matrix.go-version }}" }} - fail-fast: false - platform: "ubuntu-latest" - project-directory: "." - rootless-docker: true - run-tests: true - ryuk-disabled: false - - test-module-generator: - strategy: - matrix: - go-version: [1.22.x, 1.x] - platform: [ubuntu-latest, macos-latest, windows-latest] - uses: ./.github/workflows/ci-test-go.yml - with: - go-version: {{ "${{ matrix.go-version }}" }} - fail-fast: true - platform: {{ "${{ matrix.platform }}" }} - project-directory: "modulegen" - rootless-docker: false - run-tests: true - ryuk-disabled: false - - test-modules: - needs: test - strategy: - matrix: - go-version: [1.22.x, 1.x] - platform: [ubuntu-latest] - module: [{{ .Modules }}] - uses: ./.github/workflows/ci-test-go.yml - with: - go-version: {{ "${{ matrix.go-version }}" }} - fail-fast: false - platform: {{ "${{ matrix.platform }}" }} - project-directory: {{ "modules/${{ matrix.module }}" }} - rootless-docker: false - run-tests: {{ "${{ matrix.platform == 'ubuntu-latest' }}" }} - ryuk-disabled: false - - test-examples: - needs: test-modules - strategy: - matrix: - module: [{{ .Examples }}] - uses: ./.github/workflows/ci-test-go.yml - with: - go-version: "1.22.x" - fail-fast: true - platform: 'ubuntu-latest' - project-directory: {{ "examples/${{ matrix.module }}" }} - rootless-docker: false - run-tests: true - ryuk-disabled: false - - sonarcloud: - permissions: - contents: read # for actions/checkout to fetch code - pull-requests: read # for sonarsource/sonarcloud-github-action to determine which PR to decorate - if: {{ "${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' }}" }} - needs: test-examples - runs-on: ubuntu-latest - steps: - - name: Check out code into the Go module directory - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - # Disabling shallow clone is recommended for improving relevancy of reporting - fetch-depth: 0 - - - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - with: - name: sonarcloud - - - name: Analyze with SonarCloud - uses: sonarsource/sonarcloud-github-action@02ef91109b2d589e757aefcfb2854c2783fd7b19 # v4.0.0 - env: - GITHUB_TOKEN: {{ "${{ secrets.GITHUB_TOKEN }}" }} - SONAR_TOKEN: {{ "${{ secrets.SONAR_TOKEN }}" }} diff --git a/modulegen/internal/main.go b/modulegen/internal/main.go index 2f64c5836d..21084a5a52 100644 --- a/modulegen/internal/main.go +++ b/modulegen/internal/main.go @@ -11,7 +11,6 @@ import ( "github.com/testcontainers/testcontainers-go/modulegen/internal/sonar" "github.com/testcontainers/testcontainers-go/modulegen/internal/tools" "github.com/testcontainers/testcontainers-go/modulegen/internal/vscode" - "github.com/testcontainers/testcontainers-go/modulegen/internal/workflow" ) func Generate(moduleVar context.TestcontainersModuleVar, isModule bool) error { @@ -82,9 +81,8 @@ func GenerateFiles(ctx context.Context, tcModule context.TestcontainersModule) e // not in the new module to be added, that's why they happen after the actual // module generation projectGenerators := []ProjectGenerator{ - workflow.Generator{}, // update github ci workflow - vscode.Generator{}, // update vscode workspace - sonar.Generator{}, // update sonar-project.properties + vscode.Generator{}, // update vscode workspace + sonar.Generator{}, // update sonar-project.properties } for _, generator := range projectGenerators { diff --git a/modulegen/internal/workflow/main.go b/modulegen/internal/workflow/main.go deleted file mode 100644 index 9cf75d259c..0000000000 --- a/modulegen/internal/workflow/main.go +++ /dev/null @@ -1,40 +0,0 @@ -package workflow - -import ( - "path/filepath" - "text/template" - - "github.com/testcontainers/testcontainers-go/modulegen/internal/context" - internal_template "github.com/testcontainers/testcontainers-go/modulegen/internal/template" -) - -type Generator struct{} - -// Generate updates github ci workflow -func (g Generator) Generate(ctx context.Context) error { - rootCtx, err := context.GetRootContext() - if err != nil { - return err - } - examples, err := rootCtx.GetExamples() - if err != nil { - return err - } - modules, err := rootCtx.GetModules() - if err != nil { - return err - } - - githubWorkflowsDir := ctx.GithubWorkflowsDir() - - projectDirectories := newProjectDirectories(examples, modules) - name := "ci.yml.tmpl" - t, err := template.New(name).ParseFiles(filepath.Join("_template", name)) - if err != nil { - return err - } - - exampleFilePath := filepath.Join(githubWorkflowsDir, "ci.yml") - - return internal_template.GenerateFile(t, exampleFilePath, name, projectDirectories) -} diff --git a/modulegen/internal/workflow/types.go b/modulegen/internal/workflow/types.go deleted file mode 100644 index 77433613fb..0000000000 --- a/modulegen/internal/workflow/types.go +++ /dev/null @@ -1,17 +0,0 @@ -package workflow - -import ( - "strings" -) - -type ProjectDirectories struct { - Examples string - Modules string -} - -func newProjectDirectories(examples []string, modules []string) *ProjectDirectories { - return &ProjectDirectories{ - Examples: strings.Join(examples, ", "), - Modules: strings.Join(modules, ", "), - } -} diff --git a/modulegen/main_test.go b/modulegen/main_test.go index 10a7904b8c..72ddf59804 100644 --- a/modulegen/main_test.go +++ b/modulegen/main_test.go @@ -247,14 +247,11 @@ func TestGenerate(t *testing.T) { tmpCtx := context.New(t.TempDir()) examplesTmp := filepath.Join(tmpCtx.RootDir, "examples") examplesDocTmp := filepath.Join(tmpCtx.DocsDir(), "examples") - githubWorkflowsTmp := tmpCtx.GithubWorkflowsDir() err := os.MkdirAll(examplesTmp, 0o777) require.NoError(t, err) err = os.MkdirAll(examplesDocTmp, 0o777) require.NoError(t, err) - err = os.MkdirAll(githubWorkflowsTmp, 0o777) - require.NoError(t, err) err = copyInitialMkdocsConfig(t, tmpCtx) require.NoError(t, err) @@ -283,12 +280,7 @@ func TestGenerate(t *testing.T) { _, err = os.Stat(moduleDocFile) require.NoError(t, err) // error nil implies the file exist - mainWorkflowFile := filepath.Join(githubWorkflowsTmp, "ci.yml") - _, err = os.Stat(mainWorkflowFile) - require.NoError(t, err) // error nil implies the file exist - assertModuleDocContent(t, module, moduleDocFile) - assertModuleGithubWorkflowContent(t, mainWorkflowFile) generatedTemplatesDir := filepath.Join(examplesTmp, moduleNameLower) // do not generate examples_test.go for examples @@ -303,14 +295,11 @@ func TestGenerateModule(t *testing.T) { tmpCtx := context.New(t.TempDir()) modulesTmp := filepath.Join(tmpCtx.RootDir, "modules") modulesDocTmp := filepath.Join(tmpCtx.DocsDir(), "modules") - githubWorkflowsTmp := tmpCtx.GithubWorkflowsDir() err := os.MkdirAll(modulesTmp, 0o777) require.NoError(t, err) err = os.MkdirAll(modulesDocTmp, 0o777) require.NoError(t, err) - err = os.MkdirAll(githubWorkflowsTmp, 0o777) - require.NoError(t, err) err = copyInitialMkdocsConfig(t, tmpCtx) require.NoError(t, err) @@ -339,12 +328,7 @@ func TestGenerateModule(t *testing.T) { _, err = os.Stat(moduleDocFile) require.NoError(t, err) // error nil implies the file exist - mainWorkflowFile := filepath.Join(githubWorkflowsTmp, "ci.yml") - _, err = os.Stat(mainWorkflowFile) - require.NoError(t, err) // error nil implies the file exist - assertModuleDocContent(t, module, moduleDocFile) - assertModuleGithubWorkflowContent(t, mainWorkflowFile) generatedTemplatesDir := filepath.Join(modulesTmp, moduleNameLower) assertExamplesTestContent(t, module, filepath.Join(generatedTemplatesDir, "examples_test.go")) @@ -439,24 +423,6 @@ func assertModuleContent(t *testing.T, module context.TestcontainersModule, exam require.Equal(t, "\treturn c, nil", data[41]) } -// assert content GitHub workflow for the module -func assertModuleGithubWorkflowContent(t *testing.T, moduleWorkflowFile string) { - t.Helper() - content, err := os.ReadFile(moduleWorkflowFile) - require.NoError(t, err) - - data := sanitiseContent(content) - ctx := getTestRootContext(t) - - modulesList, err := ctx.GetModules() - require.NoError(t, err) - assert.Equal(t, " module: ["+strings.Join(modulesList, ", ")+"]", data[96]) - - examplesList, err := ctx.GetExamples() - require.NoError(t, err) - assert.Equal(t, " module: ["+strings.Join(examplesList, ", ")+"]", data[111]) -} - // assert content go.mod func assertGoModContent(t *testing.T, module context.TestcontainersModule, tcVersion string, goModFile string) { t.Helper() diff --git a/scripts/bump-go.sh b/scripts/bump-go.sh index 5ff33a5e6e..5f65e5aada 100755 --- a/scripts/bump-go.sh +++ b/scripts/bump-go.sh @@ -48,7 +48,6 @@ function main() { for f in $(find "${ROOT_DIR}/.github/workflows" -name "*.yml"); do bumpCIMatrix "${f}" "${escapedCurrentGoVersion}" "${escapedGoVersion}" done - bumpCIMatrix "${ROOT_DIR}/modulegen/_template/ci.yml.tmpl" "${escapedCurrentGoVersion}" "${escapedGoVersion}" # bump devcontainer file bumpDevcontainer "${ROOT_DIR}/.devcontainer/devcontainer.json" "${escapedCurrentGoVersion}" "${escapedGoVersion}" diff --git a/scripts/changed-modules.sh b/scripts/changed-modules.sh new file mode 100755 index 0000000000..977a3ea095 --- /dev/null +++ b/scripts/changed-modules.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash + +# exit on error, unset variables, print commands, fail on pipe errors +set -euxo pipefail + +# How to test this script, run it with the required environment variables: +# 1. A Go file from the core module is modified: +# ALL_CHANGED_FILES="examples/nginx/go.mod examples/foo/a.txt a/b/c/d/a.go" ./scripts/changed-modules.sh +# The output should be: all modules. +# +# 2. A file from a module in the modules dir is modified: +# ALL_CHANGED_FILES="modules/nginx/go.mod" ./scripts/changed-modules.sh +# The output should be: just the modules/nginx module. +# +# 3. A file from a module in the examples dir is modified: +# ALL_CHANGED_FILES="examples/nginx/go.mod" ./scripts/changed-modules.sh +# The output should be: just the examples/nginx module. +# +# 4. A Go file from the modulegen dir is modified: +# ALL_CHANGED_FILES="modulegen/a.go" ./scripts/changed-modules.sh +# The output should be: just the modulegen module. +# +# 5. A non-Go file from the core dir is modified: +# ALL_CHANGED_FILES="README.md" ./scripts/changed-modules.sh +# The output should be: all modules. +# +# 6. A file from two modules in the modules dir are modified: +# ALL_CHANGED_FILES="modules/nginx/go.mod modules/localstack/go.mod" ./scripts/changed-modules.sh +# The output should be: the modules/nginx and modules/localstack modules. +# +# 7. Files from the excluded dirs are modified: +# ALL_CHANGED_FILES="docs/a.md .vscode/a.json .devcontainer/a.json" ./scripts/changed-modules.sh +# The output should be: no modules. +# +# There is room for improvement in this script. For example, it could detect if the changes applied to the docs or the .github dirs, and then do not include any module in the list. +# But then we would need to verify the CI scripts to ensure that the job receives the correct modules to build. + +# ROOT_DIR is the root directory of the repository. +readonly ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) + +# define an array of modules that won't be included in the list +readonly excluded_modules=(".devcontainer" ".vscode" "docs") + +# modules is an array that will store the paths of all the modules in the repository. +modules=() + +# Find all go.mod files in the repository, building a list of all the available modules and examples. +for modFile in $(find "${ROOT_DIR}/modules" -name "go.mod" -not -path "${ROOT_DIR}/**/testdata/*"); do + modules+=("\"modules/$(basename "$(dirname "${modFile}")")\"") +done +for modFile in $(find "${ROOT_DIR}/examples" -name "go.mod" -not -path "${ROOT_DIR}/**/testdata/*"); do + modules+=("\"examples/$(basename "$(dirname "${modFile}")")\"") +done + +# sort modules array +IFS=$'\n' modules=($(sort <<<"${modules[*]}")) +unset IFS + +# capture the root module +readonly rootModule="\"\"" + +# capture the modulegen module +readonly modulegenModule="\"modulegen\"" + +# merge all modules and examples into a single array +allModules=(${rootModule} ${modulegenModule} "${modules[@]}") + +# sort allModules array +IFS=$'\n' allModules=($(sort <<<"${allModules[*]}")) +unset IFS + +# Get the list of modified files, retrieved from the environment variable ALL_CHANGED_FILES. +# On CI, this value will come from a Github Action retrieving the list of modified files from the pull request. +readonly modified_files=${ALL_CHANGED_FILES[@]} + +# Initialize variables +modified_modules=() + +# Check the modified files and determine which modules to build, following these rules: +# - if the modified files contain any file in the root module, include all modules in the list +# - if the modified files only contain files in one of the modules, include that module in the list +# - if the modified files only contain files in one of the examples, include that example in the list +# - if the modified files only contain files in the modulegen module, include only the modulegen module in the list +for file in $modified_files; do + if [[ $file == modules/* ]]; then + module_name=$(echo $file | cut -d'/' -f2) + if [[ ! " ${modified_modules[@]} " =~ " ${module_name} " ]]; then + modified_modules+=("\"modules/$module_name\"") + fi + elif [[ $file == examples/* ]]; then + example_name=$(echo $file | cut -d'/' -f2) + if [[ ! " ${modified_modules[@]} " =~ " ${example_name} " ]]; then + modified_modules+=("\"examples/$example_name\"") + fi + elif [[ $file == modulegen/* ]]; then + modified_modules+=("\"modulegen\"") + else + # a file from the core module is modified, so include all modules in the list and stop the loop + # check if the file is in one of the excluded modules + for exclude_module in ${excluded_modules[@]}; do + if [[ $file == $exclude_module/* ]]; then + # continue skips to the next iteration of an enclosing for, select, until, or while loop in a shell script. + # Execution continues at the loop control of the nth enclosing loop, in this case two levels up. + continue 2 + fi + done + + modified_modules=${allModules[@]} + break + fi +done + +# print all modules with this format: +# each module will be enclosed in double quotes +# each module will be separated by a comma +# the entire list will be enclosed in square brackets +echo "["$(IFS=,; echo "${modified_modules[*]}" | sed 's/ /,/g')"]"