Skip to content

Commit 27e5898

Browse files
HumairAKHumairAK
authored
Merge pull request opendatahub-io#781 from gmfrasca/preloaded-pipelines
Dynamically Update SampleConfig JSON
2 parents 64c1d81 + 28b1a5b commit 27e5898

31 files changed

+489
-45
lines changed

‎api/v1/dspipeline_types.go

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,28 @@ type DSPASpec struct {
5555
*WorkflowController `json:"workflowController,omitempty"`
5656
}
5757

58+
// +kubebuilder:validation:Pattern=`^(Managed|Removed)$`
59+
type ManagedPipelineState string
60+
61+
type ManagedPipelineOptions struct {
62+
// Set to one of the following values:
63+
//
64+
// - "Managed" : This pipeline is automatically imported.
65+
// - "Removed" : This pipeline is not automatically imported. If previously set to "Managed", setting to "Removed" does not remove existing managed pipelines but does prevent future updates from being imported.
66+
//
67+
// +kubebuilder:validation:Enum=Managed;Removed
68+
// +kubebuilder:default=Removed
69+
// +kubebuilder:validation:Optional
70+
State ManagedPipelineState `json:"state,omitempty"`
71+
}
72+
73+
type ManagedPipelinesSpec struct {
74+
// Configures whether to automatically import the InstructLab pipeline.
75+
// You must enable the trainingoperator component to run the InstructLab pipeline.
76+
// +kubebuilder:validation:Optional
77+
InstructLab *ManagedPipelineOptions `json:"instructLab,omitempty"`
78+
}
79+
5880
type APIServer struct {
5981
// Enable DS Pipelines Operator management of DSP API Server. Setting Deploy to false disables operator reconciliation. Default: true
6082
// +kubebuilder:default:=true
@@ -66,7 +88,7 @@ type APIServer struct {
6688
// +kubebuilder:default:=true
6789
// +kubebuilder:validation:Optional
6890
EnableRoute bool `json:"enableOauth"`
69-
// Include sample pipelines with the deployment of this DSP API Server. Default: true
91+
// Include the Iris sample pipeline with the deployment of this DSP API Server. Default: true
7092
// +kubebuilder:default:=false
7193
// +kubebuilder:validation:Optional
7294
EnableSamplePipeline bool `json:"enableSamplePipeline"`
@@ -82,6 +104,8 @@ type APIServer struct {
82104
ToolboxImage string `json:"toolboxImage,omitempty"`
83105
// RhelAI image used for ilab tasks in managed pipelines.
84106
RHELAIImage string `json:"rhelAIImage,omitempty"`
107+
// Enable various managed pipelines on this DSP API server.
108+
ManagedPipelines *ManagedPipelinesSpec `json:"managedPipelines,omitempty"`
85109
// Specify custom Pod resource requirements for this component.
86110
Resources *ResourceRequirements `json:"resources,omitempty"`
87111
// Specify init container resource requirements. The init container

‎api/v1/zz_generated.deepcopy.go

Lines changed: 40 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/v1alpha1/dspipeline_types.go

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,27 @@ type DSPASpec struct {
5555
*WorkflowController `json:"workflowController,omitempty"`
5656
}
5757

58+
// +kubebuilder:validation:Pattern=`^(Managed|Removed)$`
59+
type ManagedPipelineState string
60+
61+
type ManagedPipelineOptions struct {
62+
// Set to one of the following values:
63+
//
64+
// - "Managed" : This pipeline is automatically imported.
65+
// - "Removed" : This pipeline is not automatically imported. If previously set to "Managed", setting to "Removed" does not remove existing managed pipelines but does prevent future updates from being imported. //
66+
// +kubebuilder:validation:Enum=Managed;Removed
67+
// +kubebuilder:default=Removed
68+
// +kubebuilder:validation:Optional
69+
State ManagedPipelineState `json:"state,omitempty"`
70+
}
71+
72+
type ManagedPipelinesSpec struct {
73+
// Configures whether to automatically import the InstructLab pipeline.
74+
// You must enable the trainingoperator component to run the InstructLab pipeline.
75+
// +kubebuilder:validation:Optional
76+
InstructLab *ManagedPipelineOptions `json:"instructLab,omitempty"`
77+
}
78+
5879
type APIServer struct {
5980
// Enable DS Pipelines Operator management of DSP API Server. Setting Deploy to false disables operator reconciliation. Default: true
6081
// +kubebuilder:default:=true
@@ -66,7 +87,7 @@ type APIServer struct {
6687
// +kubebuilder:default:=true
6788
// +kubebuilder:validation:Optional
6889
EnableRoute bool `json:"enableOauth"`
69-
// Include sample pipelines with the deployment of this DSP API Server. Default: true
90+
// Include the Iris sample pipeline with the deployment of this DSP API Server. Default: true
7091
// +kubebuilder:default:=false
7192
// +kubebuilder:validation:Optional
7293
EnableSamplePipeline bool `json:"enableSamplePipeline"`
@@ -82,6 +103,8 @@ type APIServer struct {
82103
ToolboxImage string `json:"toolboxImage,omitempty"`
83104
// RhelAI image used for ilab tasks in managed pipelines.
84105
RHELAIImage string `json:"rhelAIImage,omitempty"`
106+
// Enable various managed pipelines on this DSP API server.
107+
ManagedPipelines *ManagedPipelinesSpec `json:"managedPipelines,omitempty"`
85108
// Specify custom Pod resource requirements for this component.
86109
Resources *ResourceRequirements `json:"resources,omitempty"`
87110
// Specify init container resource requirements. The init container

‎api/v1alpha1/zz_generated.deepcopy.go

Lines changed: 40 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/base/kustomization.yaml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,5 +167,19 @@ vars:
167167
apiVersion: v1
168168
fieldref:
169169
fieldpath: data.DSPO_APISERVER_INCLUDE_OWNERREFERENCE
170+
- name: MANAGEDPIPELINES
171+
objref:
172+
kind: ConfigMap
173+
name: dspo-parameters
174+
apiVersion: v1
175+
fieldref:
176+
fieldpath: data.MANAGEDPIPELINES
177+
- name: PLATFORMVERSION
178+
objref:
179+
kind: ConfigMap
180+
name: dspo-parameters
181+
apiVersion: v1
182+
fieldref:
183+
fieldpath: data.PLATFORMVERSION
170184
configurations:
171185
- params.yaml

‎config/base/params.env

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,5 @@ DSPO_HEALTHCHECK_DATABASE_CONNECTIONTIMEOUT=15s
1919
DSPO_HEALTHCHECK_OBJECTSTORE_CONNECTIONTIMEOUT=15s
2020
DSPO_REQUEUE_TIME=20s
2121
DSPO_APISERVER_INCLUDE_OWNERREFERENCE=true
22+
MANAGEDPIPELINES="{}"
23+
PLATFORMVERSION="v0.0.0"

‎config/configmaps/files/config.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,22 @@ Images:
1313
RuntimeGeneric: $(IMAGES_PIPELINESRUNTIMEGENERIC)
1414
Toolbox: $(IMAGES_TOOLBOX)
1515
RHELAI: $(IMAGES_RHELAI)
16+
ManagedPipelinesMetadata:
17+
Instructlab:
18+
Name: Instructlab
19+
Description: InstructLab fine-tunes models using synthetic data generation (SDG) techniques and a structured taxonomy to create diverse, high-quality training datasets.
20+
Filepath: /pipelines/instructlab.yaml
21+
VersionName: Instructlab
22+
Iris:
23+
Name: "[Demo] iris-training"
24+
Description: "[source code](https://github.com/opendatahub-io/data-science-pipelines/tree/master/samples/iris-sklearn) A simple pipeline to demonstrate a basic ML Training workflow"
25+
Filepath: /samples/iris-pipeline-compiled.yaml
26+
VersionName: "[Demo] iris-training"
1627
DSPO:
1728
HealthCheck:
1829
Database:
1930
ConnectionTimeout: $(DSPO_HEALTHCHECK_DATABASE_CONNECTIONTIMEOUT)
2031
ObjectStore:
2132
ConnectionTimeout: $(DSPO_HEALTHCHECK_OBJECTSTORE_CONNECTIONTIMEOUT)
2233
RequeueTime: $(DSPO_REQUEUE_TIME)
34+
PlatformVersion: $(PLATFORMVERSION)

‎config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml

Lines changed: 52 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,8 @@ spec:
114114
type: boolean
115115
enableSamplePipeline:
116116
default: false
117-
description: 'Include sample pipelines with the deployment of
118-
this DSP API Server. Default: true'
117+
description: 'Include the Iris sample pipeline with the deployment
118+
of this DSP API Server. Default: true'
119119
type: boolean
120120
image:
121121
description: Specify a custom image for DSP API Server.
@@ -156,6 +156,30 @@ spec:
156156
x-kubernetes-int-or-string: true
157157
type: object
158158
type: object
159+
managedPipelines:
160+
description: Enable various managed pipelines on this DSP API
161+
server.
162+
properties:
163+
instructLab:
164+
description: Configures whether to automatically import the
165+
InstructLab pipeline. You must enable the trainingoperator
166+
component to run the InstructLab pipeline.
167+
properties:
168+
state:
169+
default: Removed
170+
description: "Set to one of the following values: \n -
171+
\"Managed\" : This pipeline is automatically imported.
172+
- \"Removed\" : This pipeline is not automatically imported.
173+
If previously set to \"Managed\", setting to \"Removed\"
174+
does not remove existing managed pipelines but does
175+
prevent future updates from being imported."
176+
enum:
177+
- Managed
178+
- Removed
179+
pattern: ^(Managed|Removed)$
180+
type: string
181+
type: object
182+
type: object
159183
resources:
160184
description: Specify custom Pod resource requirements for this
161185
component.
@@ -1048,8 +1072,8 @@ spec:
10481072
type: boolean
10491073
enableSamplePipeline:
10501074
default: false
1051-
description: 'Include sample pipelines with the deployment of
1052-
this DSP API Server. Default: true'
1075+
description: 'Include the Iris sample pipeline with the deployment
1076+
of this DSP API Server. Default: true'
10531077
type: boolean
10541078
image:
10551079
description: Specify a custom image for DSP API Server.
@@ -1095,6 +1119,30 @@ spec:
10951119
description: 'Inject the archive step script. Default: true Deprecated:
10961120
DSP V1 only, will be removed in the future.'
10971121
type: boolean
1122+
managedPipelines:
1123+
description: Enable various managed pipelines on this DSP API
1124+
server.
1125+
properties:
1126+
instructLab:
1127+
description: Configures whether to automatically import the
1128+
InstructLab pipeline. You must enable the trainingoperator
1129+
component to run the InstructLab pipeline.
1130+
properties:
1131+
state:
1132+
default: Removed
1133+
description: "Set to one of the following values: \n -
1134+
\"Managed\" : This pipeline is automatically imported.
1135+
- \"Removed\" : This pipeline is not automatically imported.
1136+
If previously set to \"Managed\", setting to \"Removed\"
1137+
does not remove existing managed pipelines but does
1138+
prevent future updates from being imported.\t//"
1139+
enum:
1140+
- Managed
1141+
- Removed
1142+
pattern: ^(Managed|Removed)$
1143+
type: string
1144+
type: object
1145+
type: object
10981146
moveResultsImage:
10991147
description: 'Image used for internal artifact passing handling
11001148
within Tekton taskruns. This field specifies the image used

‎config/internal/apiserver/default/deployment.yaml.tmpl

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ spec:
1515
dspa: {{.Name}}
1616
template:
1717
metadata:
18+
annotations:
19+
configHash: {{.APIServerConfigHash}}
1820
labels:
1921
app: {{.APIServerDefaultResourceName}}
2022
component: data-science-pipelines
@@ -185,9 +187,7 @@ spec:
185187
args:
186188
- --config=/config
187189
- -logtostderr=true
188-
{{ if .APIServer.EnableSamplePipeline }}
189190
- --sampleconfig=/config/sample_config.json
190-
{{ end }}
191191
{{ if .PodToPodTLS }}
192192
- --tlsCertPath=/etc/tls/private/tls.crt
193193
- --tlsCertKeyPath=/etc/tls/private/tls.key
@@ -246,19 +246,15 @@ spec:
246246
- mountPath: /etc/tls/private
247247
name: proxy-tls
248248
{{ end }}
249-
{{ if or .APIServer.EnableSamplePipeline .CustomCABundle }}
250-
{{ if .APIServer.EnableSamplePipeline }}
251249
- name: sample-config
252250
mountPath: /config/sample_config.json
253251
subPath: sample_config.json
254252
- name: sample-pipeline
255253
mountPath: /samples/
256-
{{ end }}
257254
{{ if .CustomCABundle }}
258255
- mountPath: {{ .CustomCABundleRootMountPath }}
259256
name: ca-bundle
260257
{{ end }}
261-
{{ end }}
262258
{{ if .APIServer.EnableRoute }}
263259
- name: oauth-proxy
264260
args:
@@ -330,11 +326,9 @@ spec:
330326
configMap:
331327
name: {{ .CustomCABundle.ConfigMapName }}
332328
{{ end }}
333-
{{ if .APIServer.EnableSamplePipeline }}
334329
- name: sample-config
335330
configMap:
336331
name: sample-config-{{.Name}}
337332
- name: sample-pipeline
338333
configMap:
339334
name: sample-pipeline-{{.Name}}
340-
{{ end }}

‎config/internal/apiserver/sample-pipeline/sample-config.yaml.tmpl

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,4 @@ metadata:
88
component: data-science-pipelines
99
data:
1010
sample_config.json: |-
11-
[
12-
{
13-
"name": "[Demo] iris-training",
14-
"description": "[source code](https://github.com/opendatahub-io/data-science-pipelines/tree/master/samples/iris-sklearn) A simple pipeline to demonstrate a basic ML Training workflow",
15-
"file": "/samples/iris-pipeline-compiled.yaml"
16-
}
17-
]
11+
{{ .SampleConfigJSON }}

0 commit comments

Comments
 (0)