Skip to content

Commit d4b1a10

Browse files
mdelapenyav1vstevenh
authored
chore(ci): run lint in a separate build before running the tests (#2876)
* chore: run lint in a separate build before running the tests * fix: lint work dir * Revert "fix: lint work dir" This reverts commit 4bab0fd. * Revert "chore: run lint in a separate build before running the tests" This reverts commit c9c63bd. * chore: detect modified files and select which module to run the tests based on that * fix: read env var properly * fix: simplify * fix: single quotes * chore: remove job * fix: proper all_modified_files quotes * chore: print out modules to run * fix: do not send an array with all modules, but a comma-separated list instead * fix: do not prepend the root_dir for the modules * chore: proper runner definition * fix: use runner in the right place * chore: separate lint to a different workflow * fix: use fromJson * fix: proper jobs chain * fix: no need to lint twice * fix: remove non-existing input * fix: sanitise path * chore: only run the reaper if the core module is modified * revert: simple lint * Revert "revert: simple lint" This reverts commit e30d9a0. * revet: remove lint * Revert "revet: remove lint" This reverts commit d66f266. * chore: try with input platform * fix: forgotten platform * fix: pass a string to the reusable workflow * fix: remove missing parenthesis * fix: proper input value as string * fix: order properly * chore: simplify merging scripts * fix: proper array syntax * fix: outputs are consumed by downstream jobs that needs the producer * fix: remove extra element * chore: detemine platform in downstream job * fix: read env properly * chore: define runner in a previos step * fix: proper output format * Revert "fix: proper output format" This reverts commit 9a3f6f9. * Revert "chore: define runner in a previos step" This reverts commit c7476de. * chore: define matrix for the downstream job * fix: try with string * fix: use fromJSON * fix: items * chore: default as string * chore: pass platforms in the parent workflow * fix: single platforms must follow the array-instring format * chore: exclude certain locations from the root dir * fix: use jq to get modules count * chore: do not run sonarcloud for zero modules * chore: separate concerns * chore: format variable name * chore: no need to generate the ci.yml anymore * chore: merge lint and test workflow files again It will create a nicer UI render * fix: missing lint job in workflow * chore: run lint first * fix: update name * fix: pin the action commit Co-authored-by: Victor Martinez <[email protected]> * chore: matrix.platform * chore: run lint stage before the test one * chore: no need to read modulegen/_template in dependabot * chore: scripts is not excluded Editing the changed-modules script needs to run all * chore: exit on error, unset variables, print commands, fail on pipe errors * chore: remove verbosity in golangci-lint in the GH workflow * chore: use pinned version with 3 digits * chore: bump golangci-lint version to match makefile's * chore: omit default value for continue-on-error * chore: simplify removing the continue-on-error input * chore: bump golangci-lint GH action version * chore: do not run mod verify on new GH workers * fix: do not fail-fast modules * chore: do not skip cache for golangci-lint GH action --------- Co-authored-by: Victor Martinez <[email protected]> Co-authored-by: Steven Hartland <[email protected]>
1 parent 0d99308 commit d4b1a10

File tree

14 files changed

+256
-381
lines changed

14 files changed

+256
-381
lines changed

‎.github/dependabot.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,7 @@ version: 2
44
updates:
55
- package-ecosystem: github-actions
66
directories:
7-
- /
8-
- /modulegen/_template
7+
- /
98
schedule:
109
interval: monthly
1110
day: sunday

‎.github/workflows/ci-lint-go.yml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
name: Run lint for a Go project
2+
run-name: "${{ inputs.project-directory }}"
3+
4+
on:
5+
workflow_call:
6+
inputs:
7+
project-directory:
8+
required: true
9+
type: string
10+
default: "."
11+
description: "The directory where the Go project is located."
12+
13+
permissions:
14+
contents: read
15+
# Optional: allow read access to pull request. Use with `only-new-issues` option.
16+
# pull-requests: read
17+
18+
jobs:
19+
lint-go-project:
20+
name: "lint: ${{ inputs.project-directory }}"
21+
runs-on: 'ubuntu-latest'
22+
steps:
23+
- name: Check out code into the Go module directory
24+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
25+
26+
- name: Set up Go
27+
uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
28+
with:
29+
go-version-file: "${{ inputs.project-directory == '' && '.' || inputs.project-directory }}/go.mod"
30+
cache-dependency-path: "${{ inputs.project-directory == '' && '.' || inputs.project-directory }}/go.sum"
31+
id: go
32+
33+
- name: golangci-lint
34+
uses: golangci/golangci-lint-action@ec5d18412c0aeab7936cb16880d708ba2a64e1ae # v6.2.0
35+
with:
36+
# Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
37+
version: v1.63.4
38+
# Optional: working directory, useful for monorepos
39+
working-directory: ${{ inputs.project-directory }}
40+
41+
- name: generate
42+
working-directory: ./${{ inputs.project-directory }}
43+
shell: bash
44+
run: |
45+
make generate
46+
git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]]
47+
48+
- name: modTidy
49+
working-directory: ./${{ inputs.project-directory }}
50+
shell: bash
51+
run: |
52+
make tidy
53+
git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]]

‎.github/workflows/ci-test-go.yml‎

Lines changed: 13 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: Run tests for a Go project
2-
run-name: "${{ inputs.project-directory }} ${{ inputs.go-version }} ${{ inputs.platform }}"
2+
run-name: "${{ inputs.project-directory }} ${{ inputs.go-version }} ${{ inputs.platforms }}"
33

44
on:
55
workflow_call:
@@ -8,15 +8,11 @@ on:
88
required: true
99
type: string
1010
description: "The version of Go to use for the test."
11-
platform:
11+
platforms:
1212
required: true
1313
type: string
14-
description: "The platform to run the test on."
15-
fail-fast:
16-
required: false
17-
type: boolean
18-
default: true
19-
description: "Fail the workflow if any of the jobs fail."
14+
default: "ubuntu-latest"
15+
description: "The platforms in which the project will be run"
2016
project-directory:
2117
required: true
2218
type: string
@@ -27,11 +23,6 @@ on:
2723
type: boolean
2824
default: false
2925
description: "Run the test with rootless docker."
30-
run-tests:
31-
required: false
32-
type: boolean
33-
default: true
34-
description: "Run the tests under conditions controlled by the caller workflow."
3526
ryuk-disabled:
3627
required: false
3728
type: boolean
@@ -45,13 +36,16 @@ permissions:
4536

4637
jobs:
4738
test-go-project:
48-
name: "${{ inputs.project-directory }}/${{ inputs.platform }}/${{ inputs.go-version }}"
49-
runs-on: ${{ inputs.platform }}
50-
continue-on-error: ${{ !inputs.fail-fast }}
39+
name: "test: ${{ inputs.project-directory }}/${{ inputs.go-version }}"
40+
# Modulegen can run the tests on all platforms
5141
env:
5242
TESTCONTAINERS_RYUK_DISABLED: "${{ inputs.ryuk-disabled }}"
5343
RYUK_CONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '5m' || '60s' }}"
5444
RYUK_RECONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '30s' || '10s' }}"
45+
strategy:
46+
matrix:
47+
platform: ${{ fromJSON(inputs.platforms) }}
48+
runs-on: ${{ matrix.platform }}
5549
steps:
5650
- name: Setup rootless Docker
5751
if: ${{ inputs.rootless-docker }}
@@ -60,49 +54,15 @@ jobs:
6054
rootless: true
6155

6256
- name: Check out code into the Go module directory
63-
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
57+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
6458

6559
- name: Set up Go
66-
uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5
60+
uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
6761
with:
6862
go-version: '${{ inputs.go-version }}'
6963
cache-dependency-path: '${{ inputs.project-directory }}/go.sum'
7064
id: go
7165

72-
- name: golangci-lint
73-
if: ${{ inputs.platform == 'ubuntu-latest' }}
74-
uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1
75-
with:
76-
# Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
77-
version: v1.61.0
78-
# Optional: working directory, useful for monorepos
79-
working-directory: ${{ inputs.project-directory }}
80-
# Optional: golangci-lint command line arguments.
81-
args: --verbose
82-
# Optional: if set to true then the all caching functionality will be complete disabled,
83-
# takes precedence over all other caching options.
84-
skip-cache: true
85-
86-
- name: generate
87-
if: ${{ inputs.platform == 'ubuntu-latest' }}
88-
working-directory: ./${{ inputs.project-directory }}
89-
shell: bash
90-
run: |
91-
make generate
92-
git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]]
93-
94-
- name: modVerify
95-
working-directory: ./${{ inputs.project-directory }}
96-
run: go mod verify
97-
98-
- name: modTidy
99-
if: ${{ inputs.platform == 'ubuntu-latest' }}
100-
working-directory: ./${{ inputs.project-directory }}
101-
shell: bash
102-
run: |
103-
make tidy
104-
git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]]
105-
10666
- name: ensure compilation
10767
working-directory: ./${{ inputs.project-directory }}
10868
run: go build
@@ -118,11 +78,6 @@ jobs:
11878
fi
11979
12080
- name: go test
121-
# only run tests on linux, there are a number of things that won't allow the tests to run on anything else
122-
# many (maybe, all?) images used can only be build on Linux, they don't have Windows in their manifest, and
123-
# we can't put Windows Server in "Linux Mode" in GitHub actions
124-
# another, host mode is only available on Linux, and we have tests around that, do we skip them?
125-
if: ${{ inputs.run-tests }}
12681
working-directory: ./${{ inputs.project-directory }}
12782
timeout-minutes: 30
12883
run: make test-unit
@@ -135,7 +90,7 @@ jobs:
13590
echo "ARTIFACT_NAME=$(basename ${{ inputs.project-directory == '.' && 'core' || inputs.project-directory }})-${{ inputs.go-version }}-${{ inputs.platform }}" >> $GITHUB_ENV
13691
13792
- name: Upload SonarCloud files
138-
if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && inputs.platform == 'ubuntu-latest' && inputs.run-tests && !inputs.rootless-docker && !inputs.ryuk-disabled }}
93+
if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && matrix.platform == 'ubuntu-latest' && !inputs.rootless-docker && !inputs.ryuk-disabled }}
13994
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
14095
with:
14196
name: sonarcloud-${{ env.ARTIFACT_NAME }}

‎.github/workflows/ci.yml‎

Lines changed: 66 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -1,131 +1,124 @@
1-
# This file is autogenerated by the 'modulegen' tool.
2-
# Please update the 'ci.yml' template instead.
31
name: Main pipeline
42

53
on:
64
push:
75
branches:
86
- main
9-
paths-ignore:
10-
- '.vscode/**'
11-
- 'mkdocs.yml'
12-
- 'docs/**'
13-
- 'README.md'
147
pull_request:
15-
paths-ignore:
16-
- '.vscode/**'
17-
- 'mkdocs.yml'
18-
- 'docs/**'
19-
- 'README.md'
208

219
concurrency:
2210
group: ${{ github.workflow }}-${{ github.head_ref || github.sha }}
2311
cancel-in-progress: true
2412

2513
jobs:
14+
detect-modules:
15+
runs-on: ubuntu-latest
16+
outputs:
17+
modules: ${{ steps.set-modified-modules.outputs.modules }}
18+
modules_count: ${{ steps.set-modified-modules-count.outputs.modules_count }}
19+
steps:
20+
- name: Check out code into the Go module directory
21+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
22+
23+
- id: changed-files
24+
name: Get changed files
25+
uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45.0.4
26+
27+
- id: set-modified-modules
28+
name: Set all modified modules
29+
env:
30+
ALL_CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}"
31+
run: echo "modules=$(./scripts/changed-modules.sh)" >> $GITHUB_OUTPUT
32+
33+
- id: set-modified-modules-count
34+
name: Set all modified modules count
35+
run: echo "modules_count=$(echo ${{ toJSON(steps.set-modified-modules.outputs.modules) }} | jq '. | length')" >> $GITHUB_OUTPUT
36+
37+
- name: Print out the modules to be used
38+
run: |
39+
echo "${{ steps.set-modified-modules-count.outputs.modules_count }} modules in the build"
40+
echo "${{ steps.set-modified-modules.outputs.modules }}"
41+
42+
lint:
43+
# only run if there are modules to lint
44+
if: ${{ needs.detect-modules.outputs.modules_count > 0 }}
45+
needs:
46+
- detect-modules
47+
strategy:
48+
matrix:
49+
module: ${{ fromJSON(needs.detect-modules.outputs.modules) }}
50+
uses: ./.github/workflows/ci-lint-go.yml
51+
with:
52+
project-directory: "${{ matrix.module }}"
53+
2654
test:
55+
# only run if there are modules to test
56+
if: ${{ needs.detect-modules.outputs.modules_count > 0 }}
57+
needs:
58+
- detect-modules
59+
- lint
2760
strategy:
61+
# We don't want to fail the build the soonest but identify which modules passed and failed.
62+
fail-fast: false
2863
matrix:
2964
go-version: [1.22.x, 1.x]
30-
platform: [ubuntu-latest, macos-latest]
65+
module: ${{ fromJSON(needs.detect-modules.outputs.modules) }}
3166
uses: ./.github/workflows/ci-test-go.yml
3267
with:
3368
go-version: ${{ matrix.go-version }}
34-
fail-fast: true
35-
platform: ${{ matrix.platform }}
36-
project-directory: "."
69+
platforms: ${{ matrix.module == 'modulegen' && '["ubuntu-latest", "macos-latest", "windows-latest"]' || '["ubuntu-latest"]' }}
70+
project-directory: "${{ matrix.module }}"
3771
rootless-docker: false
38-
run-tests: ${{ matrix.platform == 'ubuntu-latest' }}
3972
ryuk-disabled: false
4073

4174
# The job below is a copy of the job above, but with ryuk disabled.
4275
# It's executed in the first stage to avoid concurrency issues.
4376
test-reaper-off:
77+
# the core module is identified by the empty string (the root path)
78+
if: ${{ contains(fromJSON(needs.detect-modules.outputs.modules), '') }}
79+
needs:
80+
- detect-modules
81+
- lint
4482
name: "Test with reaper off"
4583
strategy:
4684
matrix:
4785
go-version: [1.22.x, 1.x]
4886
uses: ./.github/workflows/ci-test-go.yml
4987
with:
5088
go-version: ${{ matrix.go-version }}
51-
fail-fast: false
52-
platform: "ubuntu-latest"
89+
platforms: '["ubuntu-latest"]'
5390
project-directory: "."
5491
rootless-docker: false
55-
run-tests: true
5692
ryuk-disabled: true
5793

5894
# The job below is a copy of the job above, but with Docker rootless.
5995
# It's executed in the first stage to avoid concurrency issues.
6096
test-rootless-docker:
97+
# the core module is identified by the empty string (the root path)
98+
if: ${{ contains(fromJSON(needs.detect-modules.outputs.modules), '') }}
99+
needs:
100+
- detect-modules
101+
- lint
61102
name: "Test with Rootless Docker"
62103
strategy:
63104
matrix:
64105
go-version: [1.22.x, 1.x]
65-
platform: [ubuntu-latest]
66106
uses: ./.github/workflows/ci-test-go.yml
67107
with:
68108
go-version: ${{ matrix.go-version }}
69-
fail-fast: false
70-
platform: "ubuntu-latest"
109+
platforms: '["ubuntu-latest"]'
71110
project-directory: "."
72111
rootless-docker: true
73-
run-tests: true
74-
ryuk-disabled: false
75-
76-
test-module-generator:
77-
strategy:
78-
matrix:
79-
go-version: [1.22.x, 1.x]
80-
platform: [ubuntu-latest, macos-latest, windows-latest]
81-
uses: ./.github/workflows/ci-test-go.yml
82-
with:
83-
go-version: ${{ matrix.go-version }}
84-
fail-fast: true
85-
platform: ${{ matrix.platform }}
86-
project-directory: "modulegen"
87-
rootless-docker: false
88-
run-tests: true
89-
ryuk-disabled: false
90-
91-
test-modules:
92-
needs: test
93-
strategy:
94-
matrix:
95-
go-version: [1.22.x, 1.x]
96-
platform: [ubuntu-latest]
97-
module: [artemis, azurite, cassandra, chroma, clickhouse, cockroachdb, compose, consul, couchbase, databend, dolt, dynamodb, elasticsearch, etcd, gcloud, grafana-lgtm, inbucket, influxdb, k3s, k6, kafka, localstack, mariadb, meilisearch, milvus, minio, mockserver, mongodb, mssql, mysql, nats, neo4j, ollama, openfga, openldap, opensearch, pinecone, postgres, pulsar, qdrant, rabbitmq, redis, redpanda, registry, surrealdb, valkey, vault, vearch, weaviate, yugabytedb]
98-
uses: ./.github/workflows/ci-test-go.yml
99-
with:
100-
go-version: ${{ matrix.go-version }}
101-
fail-fast: false
102-
platform: ${{ matrix.platform }}
103-
project-directory: modules/${{ matrix.module }}
104-
rootless-docker: false
105-
run-tests: ${{ matrix.platform == 'ubuntu-latest' }}
106-
ryuk-disabled: false
107-
108-
test-examples:
109-
needs: test-modules
110-
strategy:
111-
matrix:
112-
module: [nginx, toxiproxy]
113-
uses: ./.github/workflows/ci-test-go.yml
114-
with:
115-
go-version: "1.22.x"
116-
fail-fast: true
117-
platform: 'ubuntu-latest'
118-
project-directory: examples/${{ matrix.module }}
119-
rootless-docker: false
120-
run-tests: true
121112
ryuk-disabled: false
122113

123114
sonarcloud:
124115
permissions:
125116
contents: read # for actions/checkout to fetch code
126117
pull-requests: read # for sonarsource/sonarcloud-github-action to determine which PR to decorate
127-
if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' }}
128-
needs: test-examples
118+
if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && needs.detect-modules.outputs.modules_count > 0 }}
119+
needs:
120+
- detect-modules
121+
- test
129122
runs-on: ubuntu-latest
130123
steps:
131124
- name: Check out code into the Go module directory

0 commit comments

Comments
 (0)