Skip to content

Commit 9761a31

Browse files
hbelmirohbelmiro
authored
chore(test): fixed log collection in tests (kubeflow#11910)
Signed-off-by: Helber Belmiro <[email protected]>
1 parent e329fa3 commit 9761a31

File tree

5 files changed

+123
-14
lines changed

5 files changed

+123
-14
lines changed

‎.github/workflows/e2e-test.yml

Lines changed: 56 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -35,21 +35,27 @@ jobs:
3535
python-version: 3.9
3636

3737
- name: Create KFP cluster
38+
id: create-kfp-cluster
3839
uses: ./.github/actions/kfp-cluster
3940
with:
4041
k8s_version: ${{ matrix.k8s_version }}
42+
continue-on-error: true
4143

4244
- name: Forward API port
45+
id: forward-api-port
46+
if: ${{ (steps.create-kfp-cluster.outcome == 'success' )}}
4347
run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
44-
48+
continue-on-error: true
49+
4550
- name: Initialization tests v1
4651
id: tests
52+
if: ${{ (steps.forward-api-port.outcome == 'success' )}}
4753
working-directory: ./backend/test/initialization
4854
run: go test -v ./... -namespace kubeflow -args -runIntegrationTests=true
4955
continue-on-error: true
5056

5157
- name: Collect failed logs
52-
if: steps.tests.outcome != 'success'
58+
if: ${{ (steps.create-kfp-cluster.outcome != 'success' ) || ( steps.forward-api-port.outcome != 'success' ) || ( steps.tests.outcome != 'success' )}}
5359
run: |
5460
./.github/resources/scripts/collect-logs.sh --ns kubeflow --output /tmp/tmp_pod_log.txt
5561
exit 1
@@ -77,21 +83,27 @@ jobs:
7783
python-version: 3.9
7884

7985
- name: Create KFP cluster
86+
id: create-kfp-cluster
8087
uses: ./.github/actions/kfp-cluster
8188
with:
8289
k8s_version: ${{ matrix.k8s_version }}
90+
continue-on-error: true
8391

8492
- name: Forward API port
93+
id: forward-api-port
94+
if: ${{ (steps.create-kfp-cluster.outcome == 'success' )}}
8595
run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
96+
continue-on-error: true
8697

8798
- name: Initialization tests v2
8899
id: tests
100+
if: ${{ (steps.forward-api-port.outcome == 'success' )}}
89101
working-directory: ./backend/test/v2/initialization
90102
run: go test -v ./... -namespace kubeflow -args -runIntegrationTests=true
91103
continue-on-error: true
92104

93105
- name: Collect failed logs
94-
if: steps.tests.outcome != 'success'
106+
if: ${{ (steps.create-kfp-cluster.outcome != 'success' ) || ( steps.forward-api-port.outcome != 'success' ) || ( steps.tests.outcome != 'success' )}}
95107
run: |
96108
./.github/resources/scripts/collect-logs.sh --ns kubeflow --output /tmp/tmp_pod_log.txt
97109
exit 1
@@ -119,24 +131,33 @@ jobs:
119131
python-version: 3.9
120132

121133
- name: Create KFP cluster
134+
id: create-kfp-cluster
122135
uses: ./.github/actions/kfp-cluster
123136
with:
124137
k8s_version: ${{ matrix.k8s_version }}
138+
continue-on-error: true
125139

126140
- name: Forward API port
141+
id: forward-api-port
142+
if: ${{ (steps.create-kfp-cluster.outcome == 'success' )}}
127143
run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
144+
continue-on-error: true
128145

129146
- name: Forward MySQL port
147+
id: forward-mysql-port
148+
if: ${{ (steps.forward-api-port.outcome == 'success' )}}
130149
run: ./.github/resources/scripts/forward-port.sh "kubeflow" "mysql" 3306 3306
150+
continue-on-error: true
131151

132152
- name: API integration tests v1
133153
id: tests
154+
if: ${{ (steps.forward-mysql-port.outcome == 'success' )}}
134155
working-directory: ./backend/test/integration
135156
run: go test -v ./... -namespace kubeflow -args -runIntegrationTests=true
136157
continue-on-error: true
137158

138159
- name: Collect failed logs
139-
if: steps.tests.outcome != 'success'
160+
if: ${{ (steps.create-kfp-cluster.outcome != 'success' ) || ( steps.forward-api-port.outcome != 'success' ) || ( steps.forward-mysql-port.outcome != 'success' ) || ( steps.tests.outcome != 'success' )}}
140161
run: |
141162
./.github/resources/scripts/collect-logs.sh --ns kubeflow --output /tmp/tmp_pod_log.txt
142163
exit 1
@@ -165,25 +186,34 @@ jobs:
165186
python-version: 3.9
166187

167188
- name: Create KFP cluster
189+
id: create-kfp-cluster
168190
uses: ./.github/actions/kfp-cluster
169191
with:
170192
k8s_version: ${{ matrix.k8s_version }}
171193
pipeline_store: ${{ matrix.pipeline_store }}
194+
continue-on-error: true
172195

173196
- name: Forward API port
197+
id: forward-api-port
198+
if: ${{ (steps.create-kfp-cluster.outcome == 'success' )}}
174199
run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
200+
continue-on-error: true
175201

176202
- name: Forward MLMD port
203+
id: forward-mlmd-port
204+
if: ${{ (steps.forward-api-port.outcome == 'success' )}}
177205
run: kubectl -n kubeflow port-forward svc/metadata-grpc-service 8080:8080 &
206+
continue-on-error: true
178207

179208
- name: API integration tests v2
180209
id: tests
210+
if: ${{ (steps.forward-api-port.outcome == 'success' )}}
181211
working-directory: ./backend/test/v2/integration
182212
run: go test -v ./... -namespace kubeflow -args -runIntegrationTests=true
183213
continue-on-error: true
184214

185215
- name: Collect failed logs
186-
if: steps.tests.outcome != 'success'
216+
if: ${{ (steps.create-kfp-cluster.outcome != 'success' ) || ( steps.forward-api-port.outcome != 'success' ) || ( steps.forward-mlmd-port.outcome != 'success' ) || ( steps.tests.outcome != 'success' )}}
187217
run: |
188218
./.github/resources/scripts/collect-logs.sh --ns kubeflow --output /tmp/tmp_pod_log.txt
189219
exit 1
@@ -232,7 +262,7 @@ jobs:
232262

233263
- name: API integration tests v2
234264
id: tests
235-
if: ${{ (steps.forward-api-port.outcome == 'success' )}}
265+
if: ${{ (steps.forward-mlmd-port.outcome == 'success' )}}
236266
working-directory: ./backend/test/v2/integration
237267
run: go test -v ./... -namespace kubeflow -args -runIntegrationTests=true -useProxy=true
238268
continue-on-error: true
@@ -288,7 +318,7 @@ jobs:
288318

289319
- name: API integration tests v2
290320
id: tests
291-
if: ${{ (steps.forward-api-port.outcome == 'success' )}}
321+
if: ${{ (steps.forward-mlmd-port.outcome == 'success' )}}
292322
working-directory: ./backend/test/v2/integration
293323
run: go test -v ./... -namespace kubeflow -args -runIntegrationTests=true -cacheEnabled=false
294324
continue-on-error: true
@@ -322,28 +352,36 @@ jobs:
322352
python-version: 3.9
323353

324354
- name: Create KFP cluster
355+
id: create-kfp-cluster
325356
uses: ./.github/actions/kfp-cluster
326357
with:
327358
k8s_version: ${{ matrix.k8s_version }}
359+
continue-on-error: true
328360

329361
- name: Forward API port
362+
id: forward-api-port
363+
if: ${{ (steps.create-kfp-cluster.outcome == 'success' )}}
330364
run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
365+
continue-on-error: true
331366

332367
- name: Forward Frontend port
368+
id: forward-frontend-port
369+
if: ${{ (steps.forward-api-port.outcome == 'success' )}}
333370
run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline-ui" 3000 3000
371+
continue-on-error: true
334372

335373
- name: Build frontend integration tests image
336374
working-directory: ./test/frontend-integration-test
337375
run: docker build . -t kfp-frontend-integration-test:local
338-
continue-on-error: true
339376

340377
- name: Frontend integration tests
341378
id: tests
379+
if: ${{ (steps.forward-frontend-port.outcome == 'success' )}}
342380
run: docker run --net=host kfp-frontend-integration-test:local --remote-run true
343381
continue-on-error: true
344382

345383
- name: Collect failed logs
346-
if: steps.tests.outcome != 'success'
384+
if: ${{ (steps.create-kfp-cluster.outcome != 'success' ) || ( steps.forward-api-port.outcome != 'success' ) || ( steps.forward-frontend-port.outcome != 'success' ) || ( steps.tests.outcome != 'success' )}}
347385
run: |
348386
./.github/resources/scripts/collect-logs.sh --ns kubeflow --output /tmp/tmp_pod_log.txt
349387
exit 1
@@ -371,27 +409,35 @@ jobs:
371409
python-version: 3.9
372410

373411
- name: Create KFP cluster
412+
id: create-kfp-cluster
374413
uses: ./.github/actions/kfp-cluster
375414
with:
376415
k8s_version: ${{ matrix.k8s_version }}
416+
continue-on-error: true
377417

378418
- name: Forward API port
419+
id: forward-api-port
420+
if: ${{ (steps.create-kfp-cluster.outcome == 'success' )}}
379421
run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
422+
continue-on-error: true
380423

381424
- name: Install prerequisites
425+
if: ${{ (steps.create-kfp-cluster.outcome == 'success' )}}
382426
run: pip3 install -r ./test/sample-test/requirements.txt
383427

384428
- name: Basic sample tests - sequential
429+
if: ${{ (steps.create-kfp-cluster.outcome == 'success' )}}
385430
id: sequential-test
386431
run: python3 ./test/sample-test/sample_test_launcher.py sample_test run_test --namespace kubeflow --test-name sequential --results-gcs-dir output
387432

388433
- name: Basic sample tests - exit_handler
434+
if: ${{ (steps.create-kfp-cluster.outcome == 'success' )}}
389435
id: sample-test
390436
run: python3 ./test/sample-test/sample_test_launcher.py sample_test run_test --namespace kubeflow --test-name exit_handler --expected-result failed --results-gcs-dir output
391437
continue-on-error: true
392438

393439
- name: Collect failed logs
394-
if: ${{ (steps.sequential-test.outcome != 'success') || (steps.sample-test.outcome != 'success')}}
440+
if: ${{ (steps.create-kfp-cluster.outcome != 'success') || (steps.forward-api-port.outcome != 'success') || (steps.sequential-test.outcome != 'success') || (steps.sample-test.outcome != 'success')}}
395441
run: |
396442
./.github/resources/scripts/collect-logs.sh --ns kubeflow --output /tmp/tmp_pod_log.txt
397443
exit 1

‎.github/workflows/kfp-kubernetes-execution-tests.yml

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,51 +44,77 @@ jobs:
4444
df -h
4545
4646
- name: Create KFP cluster
47+
id: create-kfp-cluster
4748
uses: ./.github/actions/kfp-cluster
4849
with:
4950
k8s_version: ${{ matrix.k8s_version }}
51+
continue-on-error: true
5052

5153
- name: Forward API port
54+
id: forward-api-port
55+
if: ${{ (steps.create-kfp-cluster.outcome == 'success' )}}
5256
run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
57+
continue-on-error: true
5358

5459
- name: apt-get update
60+
id: apt-get-update
61+
if: ${{ (steps.forward-api-port.outcome == 'success' )}}
5562
run: sudo apt-get update
5663

5764
- name: Install protobuf-compiler
65+
id: install-protobuf-compiler
66+
if: ${{ (steps.apt-get-update.outcome == 'success' )}}
5867
run: sudo apt-get install protobuf-compiler -y
5968

6069
- name: Install setuptools
70+
id: install-setuptools
71+
if: ${{ (steps.install-protobuf-compiler.outcome == 'success' )}}
6172
run: |
6273
pip3 install setuptools
6374
pip3 freeze
6475
6576
- name: Install Wheel
77+
id: install-wheel
78+
if: ${{ (steps.install-setuptools.outcome == 'success' )}}
6679
run: pip3 install wheel==0.42.0
6780

6881
- name: Install protobuf
82+
id: install-protobuf
83+
if: ${{ (steps.install-wheel.outcome == 'success' )}}
6984
run: pip3 install protobuf==4.25.3
7085

7186
- name: Generate API proto files
87+
id: generate-api-proto-files
88+
if: ${{ (steps.install-protobuf.outcome == 'success' )}}
7289
working-directory: ./api
7390
run: make clean python
7491

7592
- name: Install kfp-pipeline-spec from source
93+
id: install-kfp-pipeline-spec
94+
if: ${{ (steps.generate-api-proto-files.outcome == 'success' )}}
7695
run: |
7796
python3 -m pip install api/v2alpha1/python
7897
7998
- name: Generate kfp-kubernetes proto files from source
99+
id: generate-kfp-kubernetes-proto-files
100+
if: ${{ (steps.install-kfp-pipeline-spec.outcome == 'success' )}}
80101
working-directory: ./kubernetes_platform
81102
run: make clean python
82103

83104
- name: Install kfp-kubernetes from source
105+
id: install-kfp-kubernetes
106+
if: ${{ (steps.generate-kfp-kubernetes-proto-files.outcome == 'success' )}}
84107
run: |
85108
pip install -e ./kubernetes_platform/python[dev]
86109
87110
- name: Install requirements
111+
id: install-requirements
112+
if: ${{ (steps.install-kfp-kubernetes.outcome == 'success' )}}
88113
run: pip install -r ./test/kfp-kubernetes-execution-tests/requirements.txt
89114

90115
- name: Run tests
91116
id: test
117+
if: ${{ (steps.install-requirements.outcome == 'success' )}}
92118
env:
93119
PULL_NUMBER: ${{ github.event.pull_request.number }}
94120
REPO_NAME: ${{ github.repository }}
@@ -99,7 +125,7 @@ jobs:
99125
continue-on-error: true
100126

101127
- name: Collect failed logs
102-
if: steps.test.outcome != 'success'
128+
if: ${{ (steps.create-kfp-cluster.outcome != 'success') || (steps.forward-api-port.outcome != 'success') || (steps.test.outcome != 'success')}}
103129
run: |
104130
./.github/resources/scripts/collect-logs.sh --ns kubeflow --output /tmp/tmp_pod_log.txt
105131
exit 1

‎.github/workflows/kfp-samples.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ jobs:
8181
id: forward-api-port
8282
if: ${{ (steps.build-sample-modelcar-image.outcome == 'success' )}}
8383
run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
84+
continue-on-error: true
8485

8586
- name: Run Samples Tests
8687
id: tests
@@ -93,7 +94,7 @@ jobs:
9394
continue-on-error: true
9495

9596
- name: Collect failed logs
96-
if: ${{ (steps.create-kfp-cluster.outcome != 'success' ) || ( steps.tests.outcome != 'success' )}}
97+
if: ${{ (steps.create-kfp-cluster.outcome != 'success' ) || (steps.forward-api-port.outcome != 'success' ) || ( steps.tests.outcome != 'success' )}}
9798
run: |
9899
./.github/resources/scripts/collect-logs.sh --ns kubeflow --output /tmp/tmp_pod_log.txt
99100
exit 1

0 commit comments

Comments
 (0)