Skip to content

Commit 6fd8ae1

Browse files
adoroszlaiadoroszlai
authored
RATIS-2210. Reduce duplication in CI workflow (#1195)
1 parent b210965 commit 6fd8ae1

File tree

4 files changed

+385
-251
lines changed

4 files changed

+385
-251
lines changed

‎.github/workflows/check.yml‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# This reusable workflow executes a single check from `dev-support/checks/`.
17+
# Before and after the check, it performs various steps based on workflow inputs.
18+
19+
name: ci-check
20+
21+
on:
22+
workflow_call:
23+
inputs:
24+
# REQUIRED
25+
script:
26+
type: string
27+
description: "Test script to run from dev-support/checks, without .sh extension"
28+
required: true
29+
30+
# OPTIONAL (ordered alphabetically)
31+
java-version:
32+
type: string
33+
description: "Java version to set up (default: 8)"
34+
default: '8'
35+
required: false
36+
37+
needs-binary-tarball:
38+
type: boolean
39+
description: "Whether to download Ratis binary tarball created by build (default: no)"
40+
default: false
41+
required: false
42+
43+
needs-maven-repo:
44+
type: boolean
45+
description: "Whether to download Ratis jars created by build (default: no)"
46+
default: false
47+
required: false
48+
49+
needs-source-tarball:
50+
type: boolean
51+
description: "Whether to download Ratis source tarball created by build (default: no)"
52+
default: false
53+
required: false
54+
55+
runner:
56+
type: string
57+
description: "GitHub Actions runner to use"
58+
default: 'ubuntu-20.04'
59+
required: false
60+
61+
script-args:
62+
type: string
63+
description: "Arguments for the test script"
64+
default: ''
65+
required: false
66+
67+
split:
68+
type: string
69+
description: "Name of split for matrix jobs, only used in display name"
70+
default: ''
71+
required: false
72+
73+
timeout-minutes:
74+
type: number
75+
description: "Job timeout in minutes (default: 30)"
76+
default: 30
77+
required: false
78+
79+
env:
80+
MAVEN_ARGS: --batch-mode --show-version
81+
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
82+
WITH_COVERAGE: true
83+
84+
jobs:
85+
check:
86+
name: ${{ (inputs.split && format('{0} ({1})', inputs.script, inputs.split)) || inputs.script }}
87+
runs-on: ${{ inputs.runner }}
88+
timeout-minutes: ${{ inputs.timeout-minutes }}
89+
steps:
90+
- name: Checkout project
91+
if: ${{ !inputs.needs-source-tarball }}
92+
uses: actions/checkout@v4
93+
94+
- name: Download source tarball
95+
if: ${{ inputs.needs-source-tarball }}
96+
uses: actions/download-artifact@v4
97+
with:
98+
name: ratis-src
99+
100+
- name: Extract source tarball
101+
if: ${{ inputs.needs-source-tarball }}
102+
run: |
103+
tar --strip-components 1 -xzvf ratis*-src.tar.gz
104+
105+
- name: Create cache for Maven dependencies
106+
if: ${{ inputs.script == 'build' }}
107+
uses: actions/cache@v4
108+
with:
109+
path: |
110+
~/.m2/repository/*/*/*
111+
!~/.m2/repository/org/apache/ratis
112+
key: maven-repo-${{ hashFiles('**/pom.xml') }}
113+
restore-keys: |
114+
maven-repo-
115+
116+
- name: Restore cache for Maven dependencies
117+
if: ${{ inputs.script != 'build' }}
118+
uses: actions/cache/restore@v4
119+
with:
120+
path: |
121+
~/.m2/repository/*/*/*
122+
!~/.m2/repository/org/apache/ratis
123+
key: maven-repo-${{ hashFiles('**/pom.xml') }}
124+
restore-keys: |
125+
maven-repo-
126+
127+
- name: Download Maven repo
128+
id: download-maven-repo
129+
if: ${{ inputs.needs-maven-repo }}
130+
uses: actions/download-artifact@v4
131+
with:
132+
name: maven-repo
133+
path: |
134+
~/.m2/repository/org/apache/ratis
135+
136+
- name: Download binary tarball
137+
if: ${{ inputs.needs-binary-tarball }}
138+
uses: actions/download-artifact@v4
139+
with:
140+
name: ratis-bin
141+
142+
- name: Extract binary tarball
143+
if: ${{ inputs.needs-binary-tarball }}
144+
run: |
145+
mkdir -p ratis-assembly/target
146+
tar xzvf ratis-*-bin.tar.gz -C ratis-assembly/target
147+
148+
- name: Setup java ${{ inputs.java-version }}
149+
if: ${{ inputs.java-version }}
150+
uses: actions/setup-java@v4
151+
with:
152+
distribution: 'temurin'
153+
java-version: ${{ inputs.java-version }}
154+
155+
- name: Execute tests
156+
run: |
157+
dev-support/checks/${{ inputs.script }}.sh ${{ inputs.script-args }}
158+
env:
159+
WITH_COVERAGE: ${{ inputs.with-coverage }}
160+
161+
- name: Summary of failures
162+
if: ${{ failure() }}
163+
run: |
164+
if [[ -s "target/${{ inputs.script }}/summary.txt" ]]; then
165+
cat target/${{ inputs.script }}/summary.txt
166+
fi
167+
168+
- name: Archive build results
169+
if: ${{ !cancelled() }}
170+
uses: actions/upload-artifact@v4
171+
with:
172+
name: ${{ (inputs.split && format('{0}-{1}', inputs.script, inputs.split)) || inputs.script }}
173+
path: target/${{ inputs.script }}
174+
continue-on-error: true
175+
176+
# The following steps are hard-coded to be run only for 'build' check,
177+
# to avoid the need for 3 more inputs.
178+
- name: Store binaries for tests
179+
if: ${{ inputs.script == 'build' && !cancelled() }}
180+
uses: actions/upload-artifact@v4
181+
with:
182+
name: ratis-bin
183+
path: |
184+
ratis-assembly/target/ratis-assembly-*-bin.tar.gz
185+
retention-days: 1
186+
187+
- name: Store source tarball for compilation
188+
if: ${{ inputs.script == 'build' && !cancelled() }}
189+
uses: actions/upload-artifact@v4
190+
with:
191+
name: ratis-src
192+
path: |
193+
ratis-assembly/target/ratis-assembly-*-src.tar.gz
194+
retention-days: 1
195+
196+
- name: Store Maven repo for tests
197+
if: ${{ inputs.script == 'build' && !cancelled() }}
198+
uses: actions/upload-artifact@v4
199+
with:
200+
name: maven-repo
201+
path: |
202+
~/.m2/repository/org/apache/ratis
203+
retention-days: 1

‎.github/workflows/ci.yml‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
name: CI
17+
18+
on:
19+
workflow_call:
20+
inputs:
21+
ref:
22+
type: string
23+
description: Ratis git ref (branch, tag or commit hash)
24+
default: ''
25+
required: false
26+
27+
jobs:
28+
build:
29+
uses: ./.github/workflows/check.yml
30+
with:
31+
script: build
32+
script-args: -Prelease
33+
timeout-minutes: 30
34+
secrets: inherit
35+
36+
compile:
37+
needs:
38+
- build
39+
strategy:
40+
matrix:
41+
java: [ 11, 17, 21 ]
42+
fail-fast: false
43+
uses: ./.github/workflows/check.yml
44+
with:
45+
java-version: ${{ matrix.java }}
46+
needs-source-tarball: true
47+
script: compile
48+
script-args: -Djavac.version=${{ matrix.java }}
49+
split: ${{ matrix.java }}
50+
timeout-minutes: 30
51+
secrets: inherit
52+
53+
repro:
54+
needs:
55+
- build
56+
uses: ./.github/workflows/check.yml
57+
with:
58+
needs-maven-repo: true
59+
script: repro
60+
script-args: -Prelease
61+
timeout-minutes: 30
62+
secrets: inherit
63+
64+
basic:
65+
strategy:
66+
matrix:
67+
check:
68+
- author
69+
- checkstyle
70+
- findbugs
71+
- rat
72+
fail-fast: false
73+
uses: ./.github/workflows/check.yml
74+
with:
75+
script: ${{ matrix.check }}
76+
timeout-minutes: 30
77+
secrets: inherit
78+
79+
unit:
80+
strategy:
81+
matrix:
82+
profile:
83+
- grpc
84+
- server
85+
- misc
86+
fail-fast: false
87+
uses: ./.github/workflows/check.yml
88+
with:
89+
script: unit
90+
script-args: -P${{ matrix.profile }}-tests
91+
split: ${{ matrix.profile }}
92+
timeout-minutes: 60
93+
secrets: inherit
94+
95+
coverage:
96+
needs:
97+
- build
98+
- unit
99+
runs-on: ubuntu-20.04
100+
timeout-minutes: 30
101+
if: (github.repository == 'apache/ratis' || github.repository == 'apache/incubator-ratis') && github.event_name != 'pull_request'
102+
steps:
103+
- name: Checkout project
104+
uses: actions/checkout@v4
105+
with:
106+
fetch-depth: 0
107+
- name: Cache for maven dependencies
108+
uses: actions/cache/restore@v4
109+
with:
110+
path: |
111+
~/.m2/repository
112+
!~/.m2/repository/org/apache/ratis
113+
key: maven-repo-${{ hashFiles('**/pom.xml') }}
114+
restore-keys: |
115+
maven-repo-
116+
- name: Setup java 17
117+
uses: actions/setup-java@v4
118+
with:
119+
distribution: 'temurin'
120+
java-version: 17
121+
- name: Download artifacts
122+
uses: actions/download-artifact@v4
123+
with:
124+
path: target/artifacts
125+
- name: Untar binaries
126+
run: |
127+
mkdir -p ratis-assembly/target
128+
tar xzvf target/artifacts/ratis-bin/ratis-assembly-*.tar.gz -C ratis-assembly/target
129+
- name: Calculate combined coverage
130+
run: ./dev-support/checks/coverage.sh
131+
- name: Upload coverage to Sonar
132+
run: ./dev-support/checks/sonar.sh
133+
env:
134+
SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
135+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
136+
- name: Archive build results
137+
uses: actions/upload-artifact@v4
138+
if: always()
139+
with:
140+
name: ${{ github.job }}
141+
path: target/${{ github.job }}

0 commit comments

Comments
 (0)