Skip to content

Commit a31f241

Browse files
roeaprtyler
roeap
authored and
rtyler
committed
ci: improve coverage collection
Signed-off-by: Robert Pack <[email protected]>
1 parent 48cf336 commit a31f241

File tree

10 files changed

+193
-101
lines changed

10 files changed

+193
-101
lines changed

‎.github/codecov.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
21
coverage:
32
status:
43
project:
@@ -12,8 +11,8 @@ coverage:
1211
if_ci_failed: error
1312
informational: false
1413

15-
1614
ignore:
1715
- "delta-inspect/"
1816
- "proofs/"
1917
- "**/*.toml"
18+
- "crates/benchmarks/"

‎.github/workflows/build.yml‎

Lines changed: 159 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,24 +18,28 @@ env:
1818
RUSTC_WRAPPER: "sccache"
1919

2020
jobs:
21+
# run various build comnfigurations, fmt, and clippy.
2122
build:
2223
strategy:
2324
fail-fast: true
2425
matrix:
2526
os:
2627
- ubuntu-latest
2728
- windows-latest
29+
- macos-latest
2830
runs-on: ${{ matrix.os }}
2931

3032
steps:
31-
- uses: actions/checkout@v3
33+
- uses: actions/checkout@v4
34+
3235
- name: Run sccache-cache
3336
uses: mozilla-actions/[email protected]
37+
3438
- name: Install minimal stable with clippy and rustfmt
3539
uses: actions-rs/toolchain@v1
3640
with:
3741
profile: default
38-
toolchain: '1.82'
42+
toolchain: "1.82"
3943
override: true
4044

4145
- name: Format
@@ -56,10 +60,34 @@ jobs:
5660
- name: Check docs
5761
run: cargo doc --no-deps --features ${{ env.DEFAULT_FEATURES }}
5862

63+
unit_test:
64+
name: Unit Tests
65+
strategy:
66+
fail-fast: true
67+
matrix:
68+
os:
69+
- ubuntu-latest
70+
- windows-latest
71+
- macos-latest
72+
runs-on: ${{ matrix.os }}
73+
74+
steps:
75+
- uses: actions/checkout@v4
76+
77+
- name: Run sccache-cache
78+
uses: mozilla-actions/[email protected]
79+
80+
- name: Install minimal stable with clippy and rustfmt
81+
uses: actions-rs/toolchain@v1
82+
with:
83+
profile: default
84+
toolchain: "1.82"
85+
override: true
86+
5987
- name: Run tests
6088
run: |
61-
gmake setup-dat
62-
cargo test --verbose --features ${{ env.DEFAULT_FEATURES }}
89+
make setup-dat
90+
cargo test --features ${{ env.DEFAULT_FEATURES }}
6391
6492
integration_test:
6593
name: Integration Tests
@@ -78,15 +106,71 @@ jobs:
78106
AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;"
79107

80108
steps:
81-
- uses: actions/checkout@v3
109+
- uses: actions/checkout@v4
110+
111+
- name: Run sccache-cache
112+
uses: mozilla-actions/[email protected]
113+
114+
- name: Install minimal stable with clippy and rustfmt
115+
uses: actions-rs/toolchain@v1
116+
with:
117+
profile: default
118+
toolchain: "1.82"
119+
override: true
120+
121+
- name: Install cargo-llvm-cov
122+
uses: taiki-e/install-action@cargo-llvm-cov
123+
124+
- name: Start emulated services
125+
run: docker compose up -d
126+
127+
- name: Run tests with rustls (default)
128+
run: |
129+
gmake setup-dat
130+
cargo llvm-cov \
131+
--features integration_test,${{ env.DEFAULT_FEATURES }} \
132+
--workspace \
133+
--exclude delta-inspect \
134+
--exclude deltalake-hdfs \
135+
--exclude deltalake-lakefs \
136+
--codecov \
137+
--output-path codecov.json
138+
139+
- name: Upload coverage to Codecov
140+
uses: codecov/codecov-action@v4
141+
with:
142+
files: codecov.json
143+
fail_ci_if_error: true
144+
env:
145+
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
146+
147+
integration_test_native_tls:
148+
name: Integration Tests (Native TLS)
149+
runs-on: ubuntu-latest
150+
env:
151+
# https://github.com/rust-lang/cargo/issues/10280
152+
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
153+
AWS_DEFAULT_REGION: "us-east-1"
154+
AWS_ACCESS_KEY_ID: deltalake
155+
AWS_SECRET_ACCESS_KEY: weloverust
156+
AWS_ENDPOINT_URL: http://localhost:4566
157+
AWS_ALLOW_HTTP: "1"
158+
AZURE_USE_EMULATOR: "1"
159+
AZURE_STORAGE_ALLOW_HTTP: "1"
160+
AZURITE_BLOB_STORAGE_URL: "http://localhost:10000"
161+
AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;"
162+
163+
steps:
164+
- uses: actions/checkout@v4
165+
82166
- name: Run sccache-cache
83167
uses: mozilla-actions/[email protected]
84168

85169
- name: Install minimal stable with clippy and rustfmt
86170
uses: actions-rs/toolchain@v1
87171
with:
88172
profile: default
89-
toolchain: '1.82'
173+
toolchain: "1.82"
90174
override: true
91175

92176
# Install Java and Hadoop for HDFS integration tests
@@ -104,16 +188,62 @@ jobs:
104188
- name: Start emulated services
105189
run: docker compose up -d
106190

107-
- name: Run tests with rustls (default)
191+
- name: Run tests with native-tls
108192
run: |
109193
gmake setup-dat
110-
cargo test --features integration_test,${{ env.DEFAULT_FEATURES }}
194+
cargo test --no-default-features --features integration_test,s3-native-tls,datafusion
111195
112-
- name: Run tests with native-tls
196+
integration_test_hdfs:
197+
name: Integration Tests (HDFS)
198+
runs-on: ubuntu-latest
199+
env:
200+
# https://github.com/rust-lang/cargo/issues/10280
201+
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
202+
203+
steps:
204+
- uses: actions/checkout@v4
205+
206+
- name: Run sccache-cache
207+
uses: mozilla-actions/[email protected]
208+
209+
- name: Install minimal stable with clippy and rustfmt
210+
uses: actions-rs/toolchain@v1
211+
with:
212+
profile: default
213+
toolchain: "1.82"
214+
override: true
215+
216+
- name: Install cargo-llvm-cov
217+
uses: taiki-e/install-action@cargo-llvm-cov
218+
219+
# Install Java and Hadoop for HDFS integration tests
220+
- uses: actions/setup-java@v4
221+
with:
222+
distribution: "temurin"
223+
java-version: "17"
224+
225+
- name: Download Hadoop
226+
run: |
227+
wget -q https://dlcdn.apache.org/hadoop/common/hadoop-3.4.0/hadoop-3.4.0.tar.gz
228+
tar -xf hadoop-3.4.0.tar.gz -C $GITHUB_WORKSPACE
229+
echo "$GITHUB_WORKSPACE/hadoop-3.4.0/bin" >> $GITHUB_PATH
230+
231+
- name: Run tests with rustls (default)
113232
run: |
114-
cargo clean
115233
gmake setup-dat
116-
cargo test --no-default-features --features integration_test,s3-native-tls,datafusion
234+
cargo llvm-cov \
235+
--features integration_test \
236+
--package deltalake-hdfs \
237+
--codecov \
238+
--output-path codecov.json
239+
240+
- name: Upload coverage to Codecov
241+
uses: codecov/codecov-action@v4
242+
with:
243+
files: codecov.json
244+
fail_ci_if_error: true
245+
env:
246+
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
117247

118248
integration_test_lakefs:
119249
name: Integration Tests (LakeFS v1.48)
@@ -123,17 +253,21 @@ jobs:
123253
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
124254

125255
steps:
126-
- uses: actions/checkout@v3
256+
- uses: actions/checkout@v4
257+
127258
- name: Run sccache-cache
128259
uses: mozilla-actions/[email protected]
129260

130261
- name: Install minimal stable with clippy and rustfmt
131262
uses: actions-rs/toolchain@v1
132263
with:
133264
profile: default
134-
toolchain: '1.82'
265+
toolchain: "1.82"
135266
override: true
136267

268+
- name: Install cargo-llvm-cov
269+
uses: taiki-e/install-action@cargo-llvm-cov
270+
137271
- name: Download Lakectl
138272
run: |
139273
wget -q https://github.com/treeverse/lakeFS/releases/download/v1.48.1/lakeFS_1.48.1_Linux_x86_64.tar.gz
@@ -146,5 +280,16 @@ jobs:
146280
- name: Run tests with rustls (default)
147281
run: |
148282
gmake setup-dat
149-
cargo test --features integration_test_lakefs,lakefs,datafusion
283+
cargo llvm-cov \
284+
--package deltalake-lakefs \
285+
--features integration_test_lakefs \
286+
--codecov \
287+
--output-path codecov.json
150288
289+
- name: Upload coverage to Codecov
290+
uses: codecov/codecov-action@v4
291+
with:
292+
files: codecov.json
293+
fail_ci_if_error: true
294+
env:
295+
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

‎.github/workflows/codecov.yml‎

Lines changed: 0 additions & 39 deletions
This file was deleted.

‎.github/workflows/dev_pr.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
contents: read
1818
pull-requests: write
1919
steps:
20-
- uses: actions/checkout@v3
20+
- uses: actions/checkout@v4
2121

2222
- name: Assign GitHub labels
2323
if: |
@@ -54,7 +54,7 @@ jobs:
5454
with:
5555
script: |
5656
const message = `**ACTION NEEDED**
57-
57+
5858
delta-rs follows the [Conventional Commits\
5959
specification](https://www.conventionalcommits.org/en/v1.0.0/) for\
6060
release automation.

‎.github/workflows/docs.yml‎

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -14,39 +14,35 @@ on:
1414
# Allow manual trigger for now
1515
workflow_dispatch:
1616

17-
env:
17+
env:
1818
IS_RELEASE: ${{ github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch' }}
1919
BUILD_ARGS: ${{ (github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch') && 'install MATURIN_EXTRA_ARGS="--manylinux off"' || 'develop' }}
2020

2121
jobs:
2222
markdown-link-check:
2323
runs-on: ubuntu-latest
2424
steps:
25-
- uses: actions/checkout@v3
26-
- uses: gaurav-nelson/github-action-markdown-link-check@v1
27-
with:
28-
config-file: docs/mlc-config.json
29-
folder-path: docs
25+
- uses: actions/checkout@v4
26+
- uses: gaurav-nelson/github-action-markdown-link-check@v1
27+
with:
28+
config-file: docs/mlc-config.json
29+
folder-path: docs
3030

3131
lint:
3232
runs-on: ubuntu-latest
3333
steps:
34-
- uses: actions/checkout@v3
34+
- uses: actions/checkout@v4
3535
- run: |
3636
cd docs
3737
make check
3838
3939
build-deploy:
40-
needs:
41-
[
42-
lint,
43-
markdown-link-check,
44-
]
40+
needs: [lint, markdown-link-check]
4541
runs-on: ubuntu-latest
4642
permissions:
4743
contents: write
4844
steps:
49-
- uses: actions/checkout@v3
45+
- uses: actions/checkout@v4
5046

5147
- name: Setup Environment
5248
uses: ./.github/actions/setup-env
@@ -80,5 +76,3 @@ jobs:
8076
git-config-name: "Github Action"
8177
git-config-email: "[email protected]"
8278
commit-message: "doc update for tag `${{ github.event.client_payload.tag || github.ref_name }}`"
83-
84-

0 commit comments

Comments
 (0)