Skip to content

Commit b6aff3c

Browse files
brianaydemirbrianaydemir
authored
Merge pull request #3051 from brianaydemir/build-and-test-fixes
Use architecture-specific runners for multi-architecture builds
2 parents 9f4d98e + 8327b9a commit b6aff3c

File tree

10 files changed

+691
-501
lines changed

10 files changed

+691
-501
lines changed

‎.github/workflows/build-and-test.yml‎

Lines changed: 324 additions & 238 deletions
Large diffs are not rendered by default.

‎.github/workflows/check-go-generate.yml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,17 @@ jobs:
1717
- name: Checkout repository
1818
uses: actions/checkout@v6
1919

20+
- name: Set Go version
21+
# Determine the string to use in setup-go's "go-version" input.
22+
id: go-version
23+
run: |
24+
version=$(grep -E '^go[[:space:]]*[0-9]+\.[0-9]+' go.mod | grep -oE '[0-9]+\.[0-9]+')
25+
echo "version=${version}.x" >> $GITHUB_OUTPUT
26+
2027
- name: Set up Go
2128
uses: actions/setup-go@v6
2229
with:
23-
go-version-file: "go.mod"
30+
go-version: "${{ steps.go-version.outputs.version }}"
2431
check-latest: true
2532
cache: false
2633

‎.github/workflows/pre-commit-linter.yml‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,17 @@ jobs:
1717
- name: Checkout repository
1818
uses: actions/checkout@v6
1919

20+
- name: Set Go version
21+
# Determine the string to use in setup-go's "go-version" input.
22+
id: go-version
23+
run: |
24+
version=$(grep -E '^go[[:space:]]*[0-9]+\.[0-9]+' go.mod | grep -oE '[0-9]+\.[0-9]+')
25+
echo "version=${version}.x" >> $GITHUB_OUTPUT
26+
2027
- name: Set up Go
2128
uses: actions/setup-go@v6
2229
with:
23-
go-version-file: "go.mod"
30+
go-version: "${{ steps.go-version.outputs.version }}"
2431
check-latest: true
2532
cache: false
2633

@@ -63,6 +70,7 @@ jobs:
6370
node-version-file: "web_ui/frontend/package.json"
6471
cache: "npm"
6572
cache-dependency-path: "web_ui/frontend/package-lock.json"
73+
check-latest: true
6674

6775
- name: Run Prettier
6876
run: |

‎.github/workflows/release.yml‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,19 @@ jobs:
3131
node-version-file: "web_ui/frontend/package.json"
3232
cache: "npm"
3333
cache-dependency-path: "web_ui/frontend/package-lock.json"
34+
check-latest: true
35+
36+
- name: Set Go version
37+
# Determine the string to use in setup-go's "go-version" input.
38+
id: go-version
39+
run: |
40+
version=$(grep -E '^go[[:space:]]*[0-9]+\.[0-9]+' go.mod | grep -oE '[0-9]+\.[0-9]+')
41+
echo "version=${version}.x" >> $GITHUB_OUTPUT
3442
3543
- name: Set up Go
3644
uses: actions/setup-go@v6
3745
with:
38-
go-version-file: "go.mod"
46+
go-version: "${{ steps.go-version.outputs.version }}"
3947
check-latest: true
4048

4149
- name: Run GoReleaser

‎.github/workflows/test-linux-pr.yaml‎

Lines changed: 0 additions & 17 deletions
This file was deleted.

‎.github/workflows/test-linux-pr.yml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
---
2+
name: Run Tests (Linux) [on pull request]
3+
4+
# On Linux, we run tests in a custom container image. However, on a pull
5+
# request, we cannot build and push a newer version of that image. Rather
6+
# than make the tests wait on the builds in build-and-test.yml, we trigger
7+
# them immediately using the most recent container image that was built.
8+
9+
on:
10+
pull_request:
11+
repository_dispatch:
12+
types:
13+
- dispatch-build
14+
workflow_dispatch:
15+
16+
jobs:
17+
run-tests:
18+
name: Linux
19+
uses: ./.github/workflows/test-linux.yml
20+
with:
21+
image: hub.opensciencegrid.org/pelican_platform/pelican-test:latest-itb

‎.github/workflows/test-linux.yml‎

Lines changed: 120 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,14 @@
1+
---
12
name: Run Tests (Linux)
23

3-
# This workflow is intended to be called from other workflows, such as
4-
# build-and-test.yml and test-linux-pr.yml.
4+
# Unlike the macOS and Windows workflows, there is no direct trigger for
5+
# this workflow, because it depends on the container image in which all the
6+
# tests will be run.
7+
#
8+
# The workflows that trigger this one include:
9+
#
10+
# - build-and-test.yml
11+
# - test-linux-pr.yml
512

613
on:
714
workflow_call:
@@ -11,7 +18,8 @@ on:
1118
type: string
1219

1320
jobs:
14-
test:
21+
run-tests:
22+
name: Linux (${{ matrix.binary_name }})
1523
runs-on: ubuntu-latest
1624
container:
1725
image: ${{ inputs.image }}
@@ -26,87 +34,112 @@ jobs:
2634
coverprofile: coverage-server.out
2735
tags: lotman
2836
steps:
29-
- name: Checkout repository
30-
uses: actions/checkout@v6
31-
with:
32-
fetch-depth: 0 # GoReleaser needs history to look up the previous tag, and so on
33-
- name: Set up Node.js
34-
uses: actions/setup-node@v6
35-
with:
36-
node-version-file: "web_ui/frontend/package.json"
37-
cache: "npm"
38-
cache-dependency-path: "web_ui/frontend/package-lock.json"
39-
- name: Cache Next.js
40-
uses: actions/cache@v4
41-
with:
42-
# Reference: https://nextjs.org/docs/pages/guides/ci-build-caching#github-actions
43-
path: |
44-
~/.npm
45-
${{ github.workspace }}/web_ui/frontend/.next/cache
46-
# Generate a new cache whenever packages or source files change.
47-
key: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.js', '**/*.jsx', '**/*.ts', '**/*.tsx') }}
48-
# If source files changed but packages didn't, rebuild from a prior cache.
49-
restore-keys: |
50-
${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-
51-
- name: Set up Go
52-
uses: actions/setup-go@v6
53-
with:
54-
go-version-file: "go.mod"
55-
check-latest: true
56-
- name: Install gotestsum
57-
run: |
58-
go install gotest.tools/gotestsum@latest
59-
# Ensure that ~/go/bin is in PATH for subsequent steps.
60-
echo "$HOME/go/bin" >> $GITHUB_PATH
61-
- name: Test
62-
env:
63-
JUNIT_FILE: junit-${{ matrix.binary_name }}.xml
64-
run: |
65-
echo "::group::Building web UI"
66-
make web-build
67-
echo "::endgroup::"
68-
gotestsum --format pkgname-and-test-fails --hide-summary=output --junitfile "$JUNIT_FILE" -- -p=4 -timeout 15m -coverpkg=./... -covermode=count -coverprofile=${{ matrix.coverprofile }} -tags=${{ matrix.tags }} ./...
69-
- name: Upload junit report
70-
if: always()
71-
uses: actions/upload-artifact@v4
72-
with:
73-
name: junit-${{ matrix.binary_name }}-linux
74-
path: junit-${{ matrix.binary_name }}.xml
75-
- name: Publish JUnit summary
76-
if: always()
77-
uses: test-summary/action@v2
78-
with:
79-
paths: junit-${{ matrix.binary_name }}.xml
80-
- name: Get total code coverage
81-
if: github.event_name == 'pull_request'
82-
id: cc
83-
run: |
84-
set -x
85-
cc_total=`go tool cover -func=${{ matrix.coverprofile }} | grep total | grep -Eo '[0-9]+\.[0-9]+'`
86-
echo "cc_total=$cc_total" >> $GITHUB_OUTPUT
87-
- name: Add coverage information to action summary
88-
if: github.event_name == 'pull_request'
89-
run: echo 'Code coverage ' ${{steps.cc.outputs.cc_total}}'%' >> $GITHUB_STEP_SUMMARY
90-
- name: Mark the checkout as safe
91-
# This is required by the GoReleaser build below and subsequent tests.
92-
run: |
93-
git config --global --add safe.directory "$GITHUB_WORKSPACE"
94-
- name: Run GoReleaser
95-
uses: goreleaser/goreleaser-action@v6
96-
with:
97-
distribution: goreleaser
98-
version: latest
99-
args: build --single-target --clean --snapshot
100-
- name: Copy the pelican binary for the end-to-end tests
101-
run: |
102-
cp dist/${{ matrix.binary_name }}_linux_amd64_v1/${{ matrix.binary_name }} ./pelican
103-
- name: Run integration tests
104-
run: ./github_scripts/citests.sh
105-
- name: Run end-to-end test for object get/put
106-
run: ./github_scripts/get_put_test.sh
107-
- name: Run end-to-end test for director stat
108-
run: ./github_scripts/stat_test.sh
109-
- name: Run end-to-end test for --version flag
110-
run: ./github_scripts/version_test.sh
111-
- name: Run end-to-end test for site-local cache standup
112-
run: ./github_scripts/site_local_cache_test.sh
37+
- name: Checkout repository
38+
uses: actions/checkout@v6
39+
with:
40+
fetch-depth: 0 # GoReleaser needs history to look up the previous tag, and so on
41+
42+
- name: Set up Node.js
43+
uses: actions/setup-node@v6
44+
with:
45+
node-version-file: "web_ui/frontend/package.json"
46+
cache: "npm"
47+
cache-dependency-path: "web_ui/frontend/package-lock.json"
48+
check-latest: true
49+
50+
- name: Cache Next.js
51+
uses: actions/cache@v4
52+
with:
53+
# Reference: https://nextjs.org/docs/pages/guides/ci-build-caching#github-actions
54+
path: |
55+
~/.npm
56+
${{ github.workspace }}/web_ui/frontend/.next/cache
57+
# Generate a new cache whenever packages or source files change.
58+
key: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.js', '**/*.jsx', '**/*.ts', '**/*.tsx', '!**/node_modules/**') }}
59+
# If source files changed but packages didn't, rebuild from a prior cache.
60+
restore-keys: |
61+
${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-
62+
63+
- name: Set Go version
64+
# Determine the string to use in setup-go's "go-version" input.
65+
id: go-version
66+
run: |
67+
version=$(grep -E '^go[[:space:]]*[0-9]+\.[0-9]+' go.mod | grep -oE '[0-9]+\.[0-9]+')
68+
echo "version=${version}.x" >> $GITHUB_OUTPUT
69+
70+
- name: Set up Go
71+
uses: actions/setup-go@v6
72+
with:
73+
go-version: "${{ steps.go-version.outputs.version }}"
74+
check-latest: true
75+
76+
- name: Install gotestsum
77+
run: |
78+
go install gotest.tools/gotestsum@latest
79+
# Ensure that ~/go/bin is in PATH for subsequent steps.
80+
echo "$HOME/go/bin" >> $GITHUB_PATH
81+
82+
- name: Run "go test"
83+
env:
84+
JUNIT_FILE: junit-${{ matrix.binary_name }}.xml
85+
run: |
86+
echo "::group::Building web UI"
87+
make web-build
88+
echo "::endgroup::"
89+
gotestsum --format pkgname-and-test-fails --hide-summary=output --junitfile "$JUNIT_FILE" -- -p=4 -timeout=15m -coverpkg=./... -covermode=count -coverprofile=${{ matrix.coverprofile }} -tags=${{ matrix.tags }} ./...
90+
91+
- name: Upload JUnit report
92+
if: always()
93+
uses: actions/upload-artifact@v4
94+
with:
95+
name: junit-${{ matrix.binary_name }}-${{ runner.os }}
96+
path: junit-${{ matrix.binary_name }}.xml
97+
overwrite: true # allow the workflow to be re-run, e.g., for flakey tests
98+
99+
- name: Publish JUnit summary
100+
if: always()
101+
uses: test-summary/action@v2
102+
with:
103+
paths: junit-${{ matrix.binary_name }}.xml
104+
105+
- name: Get total code coverage
106+
if: github.event_name == 'pull_request'
107+
id: cc
108+
run: |
109+
set -x
110+
cc_total=`go tool cover -func=${{ matrix.coverprofile }} | grep total | grep -Eo '[0-9]+\.[0-9]+'`
111+
echo "cc_total=$cc_total" >> $GITHUB_OUTPUT
112+
113+
- name: Add coverage information to action summary
114+
if: github.event_name == 'pull_request'
115+
run: echo 'Code coverage ${{ steps.cc.outputs.cc_total }}%' >> $GITHUB_STEP_SUMMARY
116+
117+
- name: Mark the checkout as safe
118+
# Because we're using a custom container image to run this workflow,
119+
# we trigger Git's "dubious ownership" warning when trying to interact
120+
# with the checkout, unless we mark it as "safe".
121+
run: |
122+
# NOTE: Refer to '${{ github.repository }}' in workflow step
123+
# definitions, but '$GITHUB_WORKSPACE' in scripts.
124+
git config --global --add safe.directory "$GITHUB_WORKSPACE"
125+
126+
- name: Run GoReleaser
127+
uses: goreleaser/goreleaser-action@v6
128+
with:
129+
distribution: goreleaser
130+
version: latest
131+
args: build --single-target --clean --snapshot
132+
133+
- name: Copy the pelican binary for the end-to-end tests
134+
run: |
135+
cp dist/${{ matrix.binary_name }}_linux_amd64_v1/${{ matrix.binary_name }} ./pelican
136+
- name: Run integration tests
137+
run: ./github_scripts/citests.sh
138+
- name: Run end-to-end test for object get/put
139+
run: ./github_scripts/get_put_test.sh
140+
- name: Run end-to-end test for director stat
141+
run: ./github_scripts/stat_test.sh
142+
- name: Run end-to-end test for --version flag
143+
run: ./github_scripts/version_test.sh
144+
- name: Run end-to-end test for site-local cache standup
145+
run: ./github_scripts/site_local_cache_test.sh

0 commit comments

Comments
 (0)