Skip to content

Commit e09d33c

Browse files
rimoliveHumairAK
rimolive
authored and
HumairAK
committed
Fix Integration tests
Signed-off-by: Ricardo M. Oliveira <[email protected]> (cherry picked from commit 0359551)
1 parent 40db6cb commit e09d33c

File tree

10 files changed

+170
-65
lines changed

10 files changed

+170
-65
lines changed

‎backend/test/integration/pipeline_api_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ func (s *PipelineApiTest) TestPipelineAPI() {
122122
time.Sleep(1 * time.Second)
123123
sequentialPipeline, err := s.pipelineClient.Create(&params.PipelineServiceCreatePipelineV1Params{
124124
Body: &model.APIPipeline{Name: "sequential", URL: &model.APIURL{
125-
PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/sequential.yaml",
125+
PipelineURL: "https://raw.githubusercontent.com/kubeflow/pipelines/refs/heads/master/backend/test/v2/resources/sequential.yaml",
126126
}},
127127
})
128128
require.Nil(t, err)
@@ -139,7 +139,7 @@ func (s *PipelineApiTest) TestPipelineAPI() {
139139
time.Sleep(1 * time.Second)
140140
argumentUrlPipeline, err := s.pipelineClient.Create(&params.PipelineServiceCreatePipelineV1Params{
141141
Body: &model.APIPipeline{URL: &model.APIURL{
142-
PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/arguments.pipeline.zip",
142+
PipelineURL: "https://github.com/kubeflow/pipelines/raw/refs/heads/master/backend/test/v2/resources/arguments.pipeline.zip",
143143
}},
144144
})
145145
require.Nil(t, err)

‎backend/test/integration/pipeline_version_api_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ func (s *PipelineVersionApiTest) TestArgoSpec() {
145145
Body: &pipeline_model.APIPipelineVersion{
146146
Name: "sequential",
147147
PackageURL: &pipeline_model.APIURL{
148-
PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/sequential.yaml",
148+
PipelineURL: "https://raw.githubusercontent.com/kubeflow/pipelines/refs/heads/master/backend/test/v2/resources/sequential.yaml",
149149
},
150150
ResourceReferences: []*pipeline_model.APIResourceReference{
151151
{
@@ -174,7 +174,7 @@ func (s *PipelineVersionApiTest) TestArgoSpec() {
174174
Body: &pipeline_model.APIPipelineVersion{
175175
Name: "arguments",
176176
PackageURL: &pipeline_model.APIURL{
177-
PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/arguments.pipeline.zip",
177+
PipelineURL: "https://github.com/kubeflow/pipelines/raw/refs/heads/master/backend/test/resources/arguments.pipeline.zip",
178178
},
179179
ResourceReferences: []*pipeline_model.APIResourceReference{
180180
{

‎backend/test/integration/upgrade_test.go

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,7 @@ func (s *UpgradeTests) PreparePipelines() {
272272
time.Sleep(1 * time.Second)
273273
sequentialPipeline, err := s.pipelineClient.Create(&pipelineParams.PipelineServiceCreatePipelineV1Params{
274274
Body: &pipeline_model.APIPipeline{Name: "sequential", URL: &pipeline_model.APIURL{
275-
PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/sequential.yaml",
275+
PipelineURL: "https://raw.githubusercontent.com/kubeflow/pipelines/refs/heads/master/backend/test/v2/resources/sequential.yaml",
276276
}},
277277
})
278278
require.Nil(t, err)
@@ -289,7 +289,7 @@ func (s *UpgradeTests) PreparePipelines() {
289289
time.Sleep(1 * time.Second)
290290
argumentUrlPipeline, err := s.pipelineClient.Create(&pipelineParams.PipelineServiceCreatePipelineV1Params{
291291
Body: &pipeline_model.APIPipeline{URL: &pipeline_model.APIURL{
292-
PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/arguments.pipeline.zip",
292+
PipelineURL: "https://github.com/kubeflow/pipelines/raw/refs/heads/master/backend/test/v2/resources/arguments.pipeline.zip",
293293
}},
294294
})
295295
require.Nil(t, err)
@@ -535,20 +535,20 @@ func (s *UpgradeTests) VerifyCreatingRunsAndJobs() {
535535

536536
/* ---------- Create a new recurring run based on the second oldest pipeline version and belonging to the second oldest experiment ---------- */
537537
createJobRequest := &jobparams.JobServiceCreateJobParams{Body: &job_model.APIJob{
538-
Name: "sequential job from pipeline version",
539-
Description: "a recurring run from an old pipeline version",
538+
Description: "a recurring run from an old pipeline version",
539+
Enabled: true,
540+
MaxConcurrency: 10,
541+
Name: "sequential job from pipeline version",
540542
ResourceReferences: []*job_model.APIResourceReference{
541543
{
542544
Key: &job_model.APIResourceKey{Type: job_model.APIResourceTypeEXPERIMENT, ID: experiments[1].ID},
543545
Relationship: job_model.APIRelationshipOWNER,
544546
},
545547
{
546-
Key: &job_model.APIResourceKey{Type: job_model.APIResourceTypePIPELINEVERSION, ID: pipelines[1].DefaultVersion.ID},
548+
Key: &job_model.APIResourceKey{Type: job_model.APIResourceTypePIPELINEVERSION, ID: pipelines[0].DefaultVersion.ID},
547549
Relationship: job_model.APIRelationshipCREATOR,
548550
},
549551
},
550-
MaxConcurrency: 10,
551-
Enabled: true,
552552
}}
553553
createdJob, err := s.jobClient.Create(createJobRequest)
554554
assert.Nil(t, err)

‎backend/test/v2/integration/pipeline_api_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ func (s *PipelineApiTest) TestPipelineAPI() {
126126
},
127127
PipelineVersion: &model.V2beta1PipelineVersion{
128128
PackageURL: &model.V2beta1URL{
129-
PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/v2/sequential.yaml",
129+
PipelineURL: "https://raw.githubusercontent.com/kubeflow/pipelines/refs/heads/master/backend/test/v2/resources/sequential-v2.yaml",
130130
},
131131
},
132132
},
@@ -141,7 +141,7 @@ func (s *PipelineApiTest) TestPipelineAPI() {
141141
assert.Equal(t, "sequential", sequentialPipelineVersions[0].DisplayName)
142142
assert.Equal(t, "sequential pipeline", sequentialPipelineVersions[0].Description)
143143
assert.Equal(t, sequentialPipeline.PipelineID, sequentialPipelineVersions[0].PipelineID)
144-
assert.Equal(t, "https://storage.googleapis.com/ml-pipeline-dataset/v2/sequential.yaml", sequentialPipelineVersions[0].PackageURL.PipelineURL)
144+
assert.Equal(t, "https://raw.githubusercontent.com/kubeflow/pipelines/refs/heads/master/backend/test/v2/resources/sequential-v2.yaml", sequentialPipelineVersions[0].PackageURL.PipelineURL)
145145

146146
/* ---------- Upload pipelines zip ---------- */
147147
time.Sleep(1 * time.Second)
@@ -164,15 +164,15 @@ func (s *PipelineApiTest) TestPipelineAPI() {
164164
Description: "1st version of argument url pipeline",
165165
PipelineID: sequentialPipeline.PipelineID,
166166
PackageURL: &model.V2beta1URL{
167-
PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/v2/arguments.pipeline.zip",
167+
PipelineURL: "https://github.com/kubeflow/pipelines/raw/refs/heads/master/backend/test/v2/resources/arguments.pipeline.zip",
168168
},
169169
},
170170
})
171171
require.Nil(t, err)
172172
assert.Equal(t, "argumentUrl-v1", argumentUrlPipelineVersion.DisplayName)
173173
assert.Equal(t, "1st version of argument url pipeline", argumentUrlPipelineVersion.Description)
174174
assert.Equal(t, argumentUrlPipeline.PipelineID, argumentUrlPipelineVersion.PipelineID)
175-
assert.Equal(t, "https://storage.googleapis.com/ml-pipeline-dataset/v2/arguments.pipeline.zip", argumentUrlPipelineVersion.PackageURL.PipelineURL)
175+
assert.Equal(t, "https://github.com/kubeflow/pipelines/raw/refs/heads/master/backend/test/v2/resources/arguments.pipeline.zip", argumentUrlPipelineVersion.PackageURL.PipelineURL)
176176

177177
/* ---------- Verify list pipeline works ---------- */
178178
pipelines, totalSize, _, err := s.pipelineClient.List(&params.PipelineServiceListPipelinesParams{})

‎backend/test/v2/integration/pipeline_version_api_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ func (s *PipelineVersionApiTest) TestPipelineSpec() {
137137
Body: &pipeline_model.V2beta1PipelineVersion{
138138
DisplayName: "sequential",
139139
PackageURL: &pipeline_model.V2beta1URL{
140-
PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/v2/sequential.yaml",
140+
PipelineURL: "https://raw.githubusercontent.com/kubeflow/pipelines/refs/heads/master/backend/test/v2/resources/sequential-v2.yaml",
141141
},
142142
PipelineID: pipelineId,
143143
},
@@ -162,7 +162,7 @@ func (s *PipelineVersionApiTest) TestPipelineSpec() {
162162
Body: &pipeline_model.V2beta1PipelineVersion{
163163
DisplayName: "arguments",
164164
PackageURL: &pipeline_model.V2beta1URL{
165-
PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/v2/arguments.pipeline.zip",
165+
PipelineURL: "https://github.com/kubeflow/pipelines/raw/refs/heads/master/backend/test/v2/resources/arguments.pipeline.zip",
166166
},
167167
PipelineID: pipelineId,
168168
},

‎backend/test/v2/integration/upgrade_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,7 @@ func (s *UpgradeTests) PreparePipelines() {
278278
Body: &pipeline_model.V2beta1PipelineVersion{
279279
DisplayName: "sequential",
280280
PackageURL: &pipeline_model.V2beta1URL{
281-
PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/v2/sequential.yaml",
281+
PipelineURL: "https://raw.githubusercontent.com/kubeflow/pipelines/refs/heads/master/backend/test/v2/resources/sequential-v2.yaml",
282282
},
283283
PipelineID: sequentialPipeline.PipelineID,
284284
},
@@ -305,7 +305,7 @@ func (s *UpgradeTests) PreparePipelines() {
305305
Body: &pipeline_model.V2beta1PipelineVersion{
306306
DisplayName: "arguments",
307307
PackageURL: &pipeline_model.V2beta1URL{
308-
PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/v2/arguments.pipeline.zip",
308+
PipelineURL: "https://github.com/kubeflow/pipelines/raw/refs/heads/master/backend/test/v2/resources/arguments.pipeline.zip",
309309
},
310310
PipelineID: argumentUrlPipeline.PipelineID,
311311
},

‎backend/test/v2/resources/sequential-v2.py

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Copyright 2023 The Kubeflow Authors
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from kfp import dsl, compiler
16+
17+
18+
@dsl.container_component
19+
def download(url: str, downloaded: dsl.OutputPath(str)):
20+
return dsl.ContainerSpec(
21+
image='google/cloud-sdk',
22+
command=['sh', '-c'],
23+
args=[f'gsutil cp {url} {downloaded}'],
24+
)
25+
26+
27+
@dsl.container_component
28+
def echo(downloaded: str):
29+
return dsl.ContainerSpec(
30+
image='library/bash',
31+
command=['sh', '-c'],
32+
args=[f'echo {downloaded}'],
33+
)
34+
35+
36+
@dsl.pipeline
37+
def sequential(url: str):
38+
echo(downloaded=download(url=url).outputs['downloaded'])
39+
40+
41+
if __name__ == '__main__':
42+
compiler.Compiler().compile(
43+
sequential, package_path=__file__.replace('.py', '.yaml'))

‎backend/test/v2/resources/sequential-v2.yaml

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
# PIPELINE DEFINITION
2+
# Name: sequential
3+
# Inputs:
4+
# url: str
5+
components:
6+
comp-download:
7+
executorLabel: exec-download
8+
inputDefinitions:
9+
parameters:
10+
url:
11+
parameterType: STRING
12+
outputDefinitions:
13+
parameters:
14+
downloaded:
15+
parameterType: STRING
16+
comp-echo:
17+
executorLabel: exec-echo
18+
inputDefinitions:
19+
parameters:
20+
downloaded:
21+
parameterType: STRING
22+
deploymentSpec:
23+
executors:
24+
exec-download:
25+
container:
26+
args:
27+
- gsutil cp {{$.inputs.parameters['url']}} {{$.outputs.parameters['downloaded'].output_file}}
28+
command:
29+
- sh
30+
- -c
31+
image: google/cloud-sdk
32+
exec-echo:
33+
container:
34+
args:
35+
- echo {{$.inputs.parameters['downloaded']}}
36+
command:
37+
- sh
38+
- -c
39+
image: library/bash
40+
pipelineInfo:
41+
name: sequential
42+
root:
43+
dag:
44+
tasks:
45+
download:
46+
cachingOptions:
47+
enableCache: true
48+
componentRef:
49+
name: comp-download
50+
inputs:
51+
parameters:
52+
url:
53+
componentInputParameter: url
54+
taskInfo:
55+
name: download
56+
echo:
57+
cachingOptions:
58+
enableCache: true
59+
componentRef:
60+
name: comp-echo
61+
dependentTasks:
62+
- download
63+
inputs:
64+
parameters:
65+
downloaded:
66+
taskOutputParameter:
67+
outputParameterKey: downloaded
68+
producerTask: download
69+
taskInfo:
70+
name: echo
71+
inputDefinitions:
72+
parameters:
73+
url:
74+
parameterType: STRING
75+
schemaVersion: 2.1.0
76+
sdkVersion: kfp-2.0.0-beta.13

‎backend/test/v2/resources/sequential.py

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,28 +14,19 @@
1414

1515
from kfp import dsl, compiler
1616

17-
18-
@dsl.container_component
19-
def download(url: str, downloaded: dsl.OutputPath(str)):
20-
return dsl.ContainerSpec(
21-
image='google/cloud-sdk',
22-
command=['sh', '-c'],
23-
args=[f'gsutil cp {url} {downloaded}'],
24-
)
25-
26-
2717
@dsl.container_component
28-
def echo(downloaded: str):
18+
def echo(message: str):
2919
return dsl.ContainerSpec(
3020
image='library/bash',
3121
command=['sh', '-c'],
32-
args=[f'echo {downloaded}'],
22+
args=[f'echo {message}'],
3323
)
3424

3525

3626
@dsl.pipeline
37-
def sequential(url: str):
38-
echo(downloaded=download(url=url).outputs['downloaded'])
27+
def sequential(param1: str, param2: str):
28+
echo(message=param1)
29+
echo(message=param2)
3930

4031

4132
if __name__ == '__main__':

0 commit comments

Comments
 (0)