Skip to content

Commit 11a9a7c

Browse files
gmfrascagmfrasca
committed
Dynamically Generate Sample Config
- Allows for configuration of per-DSPA Managed Pipelines Signed-off-by: Giulio Frasca <[email protected]>
1 parent 64c1d81 commit 11a9a7c

14 files changed

+301
-8
lines changed

‎api/v1/dspipeline_types.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,17 @@ type DSPASpec struct {
5555
*WorkflowController `json:"workflowController,omitempty"`
5656
}
5757

58+
type ManagedPipelines struct {
59+
// Include instructlab multi-phase training pipelines with the deployment of this DSP API Server. Default: true
60+
// Applicable values:"Managed" or "Removed"
61+
// +kubebuilder:validation:Optional
62+
EnableInstructLabPipeline string `json:"enableInstructLabPipeline,omitempty"`
63+
// Include sample pipelines with the deployment of this DSP API Server. Default: true
64+
// Applicable values:"Managed" or "Removed"
65+
// +kubebuilder:validation:Optional
66+
EnableIrisPipeline string `json:"enableIrisPipeline,omitempty"`
67+
}
68+
5869
type APIServer struct {
5970
// Enable DS Pipelines Operator management of DSP API Server. Setting Deploy to false disables operator reconciliation. Default: true
6071
// +kubebuilder:default:=true
@@ -82,6 +93,8 @@ type APIServer struct {
8293
ToolboxImage string `json:"toolboxImage,omitempty"`
8394
// RhelAI image used for ilab tasks in managed pipelines.
8495
RHELAIImage string `json:"rhelAIImage,omitempty"`
96+
// Enable various pipelines with the deployment of this DSP API server.
97+
ManagedPipelines *ManagedPipelines `json:"managedPipelines,omitempty"`
8598
// Specify custom Pod resource requirements for this component.
8699
Resources *ResourceRequirements `json:"resources,omitempty"`
87100
// Specify init container resource requirements. The init container

‎api/v1/zz_generated.deepcopy.go

Lines changed: 20 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/v1alpha1/dspipeline_types.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,17 @@ type DSPASpec struct {
5555
*WorkflowController `json:"workflowController,omitempty"`
5656
}
5757

58+
type ManagedPipelines struct {
59+
// Include instructlab multi-phase training pipelines with the deployment of this DSP API Server. Default: true
60+
// Applicable values:"Managed" or "Removed"
61+
// +kubebuilder:validation:Optional
62+
EnableInstructLabPipeline string `json:"enableInstructLabPipeline,omitempty"`
63+
// Include sample pipelines with the deployment of this DSP API Server. Default: true
64+
// Applicable values:"Managed" or "Removed"
65+
// +kubebuilder:validation:Optional
66+
EnableIrisPipeline string `json:"enableIrisPipeline,omitempty"`
67+
}
68+
5869
type APIServer struct {
5970
// Enable DS Pipelines Operator management of DSP API Server. Setting Deploy to false disables operator reconciliation. Default: true
6071
// +kubebuilder:default:=true
@@ -82,6 +93,8 @@ type APIServer struct {
8293
ToolboxImage string `json:"toolboxImage,omitempty"`
8394
// RhelAI image used for ilab tasks in managed pipelines.
8495
RHELAIImage string `json:"rhelAIImage,omitempty"`
96+
// Enable various pipelines with the deployment of this DSP API server.
97+
ManagedPipelines *ManagedPipelines `json:"managedPipelines,omitempty"`
8598
// Specify custom Pod resource requirements for this component.
8699
Resources *ResourceRequirements `json:"resources,omitempty"`
87100
// Specify init container resource requirements. The init container

‎api/v1alpha1/zz_generated.deepcopy.go

Lines changed: 20 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/base/kustomization.yaml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,5 +167,19 @@ vars:
167167
apiVersion: v1
168168
fieldref:
169169
fieldpath: data.DSPO_APISERVER_INCLUDE_OWNERREFERENCE
170+
- name: MANAGEDPIPELINES
171+
objref:
172+
kind: ConfigMap
173+
name: dspo-parameters
174+
apiVersion: v1
175+
fieldref:
176+
fieldpath: data.MANAGEDPIPELINES
177+
- name: PLATFORMVERSION
178+
objref:
179+
kind: ConfigMap
180+
name: dspo-parameters
181+
apiVersion: v1
182+
fieldref:
183+
fieldpath: data.PLATFORMVERSION
170184
configurations:
171185
- params.yaml

‎config/base/params.env

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,5 @@ DSPO_HEALTHCHECK_DATABASE_CONNECTIONTIMEOUT=15s
1919
DSPO_HEALTHCHECK_OBJECTSTORE_CONNECTIONTIMEOUT=15s
2020
DSPO_REQUEUE_TIME=20s
2121
DSPO_APISERVER_INCLUDE_OWNERREFERENCE=true
22+
MANAGEDPIPELINES="{}"
23+
PLATFORMVERSION="v0.0.0"

‎config/configmaps/files/config.yaml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,25 @@ Images:
1313
RuntimeGeneric: $(IMAGES_PIPELINESRUNTIMEGENERIC)
1414
Toolbox: $(IMAGES_TOOLBOX)
1515
RHELAI: $(IMAGES_RHELAI)
16+
Samples:
17+
Iris: /pipelines/iris-compiled.yaml
18+
Instructlab: /pipelines/instructlab.yaml
19+
ManagedPipelinesMetadata:
20+
Instructlab:
21+
Name: "[InstructLab] LLM Training Pipeline"
22+
Description:
23+
Filepath: /pipelines/instructlab.yaml
24+
VersionName: "[InstructLab] LLM Training Pipeline - $(PLATFORMVERSION)"
25+
Iris:
26+
Name: "[Demo] iris-training"
27+
Description: "[source code](https://github.com/opendatahub-io/data-science-pipelines/tree/master/samples/iris-sklearn) A simple pipeline to demonstrate a basic ML Training workflow"
28+
Filepath: /samples/iris-pipeline-compiled.yaml
29+
VersionName: "[Demo] iris-training - $(PLATFORMVERSION)"
1630
DSPO:
1731
HealthCheck:
1832
Database:
1933
ConnectionTimeout: $(DSPO_HEALTHCHECK_DATABASE_CONNECTIONTIMEOUT)
2034
ObjectStore:
2135
ConnectionTimeout: $(DSPO_HEALTHCHECK_OBJECTSTORE_CONNECTIONTIMEOUT)
2236
RequeueTime: $(DSPO_REQUEUE_TIME)
37+
PlatformVersion: $(PLATFORMVERSION)

‎config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,21 @@ spec:
156156
x-kubernetes-int-or-string: true
157157
type: object
158158
type: object
159+
managedPipelines:
160+
description: Enable various pipelines with the deployment of this
161+
DSP API server.
162+
properties:
163+
enableInstructLabPipeline:
164+
description: 'Include instructlab multi-phase training pipelines
165+
with the deployment of this DSP API Server. Default: true
166+
Applicable values:"Managed" or "Removed"'
167+
type: string
168+
enableIrisPipeline:
169+
description: 'Include sample pipelines with the deployment
170+
of this DSP API Server. Default: true Applicable values:"Managed"
171+
or "Removed"'
172+
type: string
173+
type: object
159174
resources:
160175
description: Specify custom Pod resource requirements for this
161176
component.
@@ -1095,6 +1110,21 @@ spec:
10951110
description: 'Inject the archive step script. Default: true Deprecated:
10961111
DSP V1 only, will be removed in the future.'
10971112
type: boolean
1113+
managedPipelines:
1114+
description: Enable various pipelines with the deployment of this
1115+
DSP API server.
1116+
properties:
1117+
enableInstructLabPipeline:
1118+
description: 'Include instructlab multi-phase training pipelines
1119+
with the deployment of this DSP API Server. Default: true
1120+
Applicable values:"Managed" or "Removed"'
1121+
type: string
1122+
enableIrisPipeline:
1123+
description: 'Include sample pipelines with the deployment
1124+
of this DSP API Server. Default: true Applicable values:"Managed"
1125+
or "Removed"'
1126+
type: string
1127+
type: object
10981128
moveResultsImage:
10991129
description: 'Image used for internal artifact passing handling
11001130
within Tekton taskruns. This field specifies the image used

‎config/internal/apiserver/sample-pipeline/sample-config.yaml.tmpl

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,4 @@ metadata:
88
component: data-science-pipelines
99
data:
1010
sample_config.json: |-
11-
[
12-
{
13-
"name": "[Demo] iris-training",
14-
"description": "[source code](https://github.com/opendatahub-io/data-science-pipelines/tree/master/samples/iris-sklearn) A simple pipeline to demonstrate a basic ML Training workflow",
15-
"file": "/samples/iris-pipeline-compiled.yaml"
16-
}
17-
]
11+
{{ .SampleConfigJSON }}

‎config/manager/manager.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,10 @@ spec:
7979
# It must always be enabled in production
8080
- name: DSPO_APISERVER_INCLUDE_OWNERREFERENCE
8181
value: $(DSPO_APISERVER_INCLUDE_OWNERREFERENCE)
82+
- name: MANAGEDPIPELINES
83+
value: $(MANAGEDPIPELINES)
84+
- name: PLATFORMVERSION
85+
value: $(PLATFORMVERSION)
8286
securityContext:
8387
allowPrivilegeEscalation: false
8488
capabilities:

0 commit comments

Comments
 (0)