Skip to content

Commit ff92a54

Browse files
Sriram Ananthakrishnanvsin12
Sriram Ananthakrishnan
authored and
vsin12
committed
Added - Support for Dataflow Integration and Storage Mounts with Pipelines
1 parent 396284d commit ff92a54

12 files changed

+1801
-127
lines changed

‎examples/datascience/pipeline/pipeline.tf‎

Lines changed: 319 additions & 72 deletions
Large diffs are not rendered by default.

‎internal/integrationtest/datascience_pipeline_run_test.go‎

Lines changed: 93 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -73,12 +73,28 @@ var (
7373
"step_override_details": acctest.RepresentationGroup{RepType: acctest.Optional, Group: pipelineRunStepOverrideDetailsContainerRepresentation},
7474
}
7575

76+
pipelineRunDataflowRepresentation = map[string]interface{}{
77+
"compartment_id": acctest.Representation{RepType: acctest.Required, Create: `${var.compartment_id}`},
78+
"pipeline_id": acctest.Representation{RepType: acctest.Required, Create: `${oci_datascience_pipeline.test_pipeline.id}`},
79+
"project_id": acctest.Representation{RepType: acctest.Required, Create: `${oci_datascience_project.test_project.id}`},
80+
"configuration_override_details": acctest.RepresentationGroup{RepType: acctest.Optional, Group: pipelineRunDataflowConfigurationOverrideDetailsRepresentation},
81+
"display_name": acctest.Representation{RepType: acctest.Optional, Create: `displayName`, Update: `displayName2`},
82+
"freeform_tags": acctest.Representation{RepType: acctest.Optional, Create: map[string]string{"Department": "Finance"}, Update: map[string]string{"Department": "Accounting"}},
83+
"delete_related_job_runs": acctest.Representation{RepType: acctest.Required, Create: `true`, Update: `true`},
84+
"log_configuration_override_details": acctest.RepresentationGroup{RepType: acctest.Optional, Group: pipelineRunLogConfigurationOverrideDetailsRepresentation},
85+
"step_override_details": acctest.RepresentationGroup{RepType: acctest.Optional, Group: pipelineRunStepOverrideDetailsDataflowRepresentation},
86+
}
87+
7688
pipelineRunConfigurationOverrideDetailsRepresentation = map[string]interface{}{
7789
"type": acctest.Representation{RepType: acctest.Required, Create: `DEFAULT`},
7890
"command_line_arguments": acctest.Representation{RepType: acctest.Optional, Create: `commandLineArgumentsOverriden`},
7991
"environment_variables": acctest.Representation{RepType: acctest.Optional, Create: map[string]string{"environmentVariablesOverriden": "environmentVariablesOverriden"}},
8092
"maximum_runtime_in_minutes": acctest.Representation{RepType: acctest.Optional, Create: `10`},
8193
}
94+
pipelineRunDataflowConfigurationOverrideDetailsRepresentation = map[string]interface{}{
95+
"type": acctest.Representation{RepType: acctest.Required, Create: `DEFAULT`},
96+
"maximum_runtime_in_minutes": acctest.Representation{RepType: acctest.Optional, Create: `60`},
97+
}
8298
pipelineRunLogConfigurationOverrideDetailsRepresentation = map[string]interface{}{
8399
"enable_auto_log_creation": acctest.Representation{RepType: acctest.Optional, Create: `false`},
84100
"enable_logging": acctest.Representation{RepType: acctest.Optional, Create: `true`},
@@ -95,19 +111,52 @@ var (
95111
"step_name": acctest.Representation{RepType: acctest.Required, Create: `stepNameContainer`},
96112
"step_container_configuration_details": acctest.RepresentationGroup{RepType: acctest.Optional, Group: DatasciencePipelineRunStepOverrideDetailsContainerConfigurationDetailsRepresentation},
97113
}
114+
pipelineRunStepOverrideDetailsDataflowRepresentation = map[string]interface{}{
115+
"step_configuration_details": acctest.RepresentationGroup{RepType: acctest.Required, Group: pipelineRunStepOverrideDetailsStepConfigurationDetailsRepresentationDataflow},
116+
"step_name": acctest.Representation{RepType: acctest.Required, Create: `stepNameDataflow`},
117+
"step_dataflow_configuration_details": acctest.RepresentationGroup{RepType: acctest.Optional, Group: DatasciencePipelineRunStepOverrideDetailsStepDataflowConfigurationDetailsRepresentation},
118+
}
98119
pipelineRunStepOverrideDetailsStepConfigurationDetailsRepresentation = map[string]interface{}{
99120
"command_line_arguments": acctest.Representation{RepType: acctest.Optional, Create: `commandLineArgumentsOverriden`},
100121
"environment_variables": acctest.Representation{RepType: acctest.Optional, Create: map[string]string{"environmentVariables": "environmentVariablesOverriden"}},
101122
"maximum_runtime_in_minutes": acctest.Representation{RepType: acctest.Optional, Create: `10`},
102123
}
124+
pipelineRunStepOverrideDetailsStepConfigurationDetailsRepresentationDataflow = map[string]interface{}{
125+
"maximum_runtime_in_minutes": acctest.Representation{RepType: acctest.Optional, Create: `60`},
126+
}
103127
DatasciencePipelineRunStepOverrideDetailsContainerConfigurationDetailsRepresentation = map[string]interface{}{
104128
"container_type": acctest.Representation{RepType: acctest.Required, Create: `OCIR_CONTAINER`},
105-
"image": acctest.Representation{RepType: acctest.Required, Create: `iad.ocir.io/ociodscdev/nested-rp-public-python-sdk-1:1.0`},
106-
"cmd": acctest.Representation{RepType: acctest.Optional, Create: []string{`hello_world.py`}},
107-
"entrypoint": acctest.Representation{RepType: acctest.Optional, Create: []string{`python3`}},
129+
"image": acctest.Representation{RepType: acctest.Required, Create: `iad.ocir.io/idtlxnfdweil/byod-hello-world:1`},
130+
"cmd": acctest.Representation{RepType: acctest.Optional, Create: []string{``}},
131+
"entrypoint": acctest.Representation{RepType: acctest.Optional, Create: []string{``}},
108132
"image_digest": acctest.Representation{RepType: acctest.Optional, Create: ``},
109133
"image_signature_id": acctest.Representation{RepType: acctest.Optional, Create: ``},
110134
}
135+
DatasciencePipelineRunStepOverrideDetailsStepDataflowConfigurationDetailsRepresentation = map[string]interface{}{
136+
"configuration": acctest.Representation{RepType: acctest.Optional, Create: map[string]string{"dataflow.auth": "3.1.0"}},
137+
"driver_shape": acctest.Representation{RepType: acctest.Optional, Create: `VM.Standard.E5.Flex`},
138+
"driver_shape_config_details": acctest.RepresentationGroup{RepType: acctest.Optional, Group: DatasciencePipelineStepDetailsStepDataflowConfigurationDetailsDriverShapeConfigDetailsRepresentation},
139+
"executor_shape": acctest.Representation{RepType: acctest.Optional, Create: `VM.Standard.E5.Flex`},
140+
"executor_shape_config_details": acctest.RepresentationGroup{RepType: acctest.Optional, Group: DatasciencePipelineStepDetailsStepDataflowConfigurationDetailsExecutorShapeConfigDetailsRepresentation},
141+
"logs_bucket_uri": acctest.Representation{RepType: acctest.Optional, Create: `oci://xuejuzha-test@idtlxnfdweil/`},
142+
"num_executors": acctest.Representation{RepType: acctest.Optional, Create: `1`},
143+
"warehouse_bucket_uri": acctest.Representation{RepType: acctest.Optional, Create: `oci://xuejuzha-test@idtlxnfdweil/`},
144+
}
145+
DatasciencePipelineRunStepOverrideDetailsStepDataflowConfigurationDetailsDriverShapeConfigDetailsRepresentation = map[string]interface{}{
146+
"memory_in_gbs": acctest.Representation{RepType: acctest.Optional, Create: `8.0`},
147+
"ocpus": acctest.Representation{RepType: acctest.Optional, Create: `1.0`},
148+
}
149+
DatasciencePipelineRunStepOverrideDetailsStepDataflowConfigurationDetailsExecutorShapeConfigDetailsRepresentation = map[string]interface{}{
150+
"memory_in_gbs": acctest.Representation{RepType: acctest.Optional, Create: `8.0`},
151+
"ocpus": acctest.Representation{RepType: acctest.Optional, Create: `1.0`},
152+
}
153+
154+
// DatasciencePipelineRunResourceDependencies = acctest.GenerateDataSourceFromRepresentationMap("oci_core_shapes", "test_shapes", acctest.Required, acctest.Create, CoreShapeDataSourceRepresentation) +
155+
// acctest.GenerateResourceFromRepresentationMap("oci_core_subnet", "test_subnet", acctest.Required, acctest.Create, CoreSubnetRepresentation) +
156+
// acctest.GenerateResourceFromRepresentationMap("oci_core_vcn", "test_vcn", acctest.Required, acctest.Create, CoreVcnRepresentation) +
157+
// acctest.GenerateResourceFromRepresentationMap("oci_dataflow_application", "test_application", acctest.Required, acctest.Create, DataflowApplicationRepresentation) +
158+
// acctest.GenerateResourceFromRepresentationMap("oci_datascience_job", "test_job", acctest.Required, acctest.Create, DatascienceJobRepresentation) +
159+
// acctest.GenerateResourceFromRepresentationMap("oci_datascience_pipeline", "test_pipeline", acctest.Required, acctest.Create, DatasciencePipelineRepresentation)
111160

112161
PipelineRunResourceDependencies = acctest.GenerateResourceFromRepresentationMap("oci_datascience_pipeline", "test_pipeline", acctest.Required, acctest.Create, pipelineRepresentation) +
113162
acctest.GenerateResourceFromRepresentationMap("oci_datascience_project", "test_project", acctest.Required, acctest.Create, DatascienceProjectRepresentation) +
@@ -118,6 +167,11 @@ var (
118167
acctest.GenerateResourceFromRepresentationMap("oci_datascience_project", "test_project", acctest.Required, acctest.Create, DatascienceProjectRepresentation) +
119168
acctest.GenerateResourceFromRepresentationMap("oci_logging_log_group", "terraform_test_custom_log_group", acctest.Required, acctest.Create, pipelineLogGroupRepresentation) +
120169
acctest.GenerateResourceFromRepresentationMap("oci_logging_log", "terraform_test_custom_log", acctest.Required, acctest.Create, pipelineLogRepresentation)
170+
171+
PipelineRunDataflowResourceDependencies = acctest.GenerateResourceFromRepresentationMap("oci_datascience_pipeline", "test_pipeline", acctest.Optional, acctest.Create, pipelineRepresentationDataflow) +
172+
acctest.GenerateResourceFromRepresentationMap("oci_datascience_project", "test_project", acctest.Required, acctest.Create, DatascienceProjectRepresentation) +
173+
acctest.GenerateResourceFromRepresentationMap("oci_logging_log_group", "terraform_test_custom_log_group", acctest.Required, acctest.Create, pipelineLogGroupRepresentation) +
174+
acctest.GenerateResourceFromRepresentationMap("oci_logging_log", "terraform_test_custom_log", acctest.Required, acctest.Create, pipelineLogRepresentation)
121175
)
122176

123177
// issue-routing-tag: datascience/default
@@ -144,6 +198,39 @@ func TestDatasciencePipelineRunResource_basic(t *testing.T) {
144198

145199
acctest.ResourceTest(t, testAccCheckDatasciencePipelineRunDestroy, []resource.TestStep{
146200
// Step 0 - Verify Create Pipeline Run with Container
201+
{
202+
Config: config + compartmentIdVariableStr + PipelineRunDataflowResourceDependencies +
203+
acctest.GenerateResourceFromRepresentationMap("oci_datascience_pipeline_run", "test_pipeline_run", acctest.Optional, acctest.Create, pipelineRunDataflowRepresentation),
204+
Check: acctest.ComposeAggregateTestCheckFuncWrapper(
205+
resource.TestCheckResourceAttr(resourceName, "compartment_id", compartmentId),
206+
resource.TestCheckResourceAttrSet(resourceName, "pipeline_id"),
207+
resource.TestCheckResourceAttrSet(resourceName, "project_id"),
208+
resource.TestCheckResourceAttrSet(resourceName, "id"),
209+
resource.TestCheckResourceAttr(resourceName, "step_override_details.#", "1"),
210+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_dataflow_configuration_details.#", "1"),
211+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_name", "stepNameDataflow"),
212+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_dataflow_configuration_details.#", "1"),
213+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_dataflow_configuration_details.0.driver_shape", "VM.Standard.E5.Flex"),
214+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_dataflow_configuration_details.0.driver_shape_config_details.#", "1"),
215+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_dataflow_configuration_details.0.driver_shape_config_details.0.memory_in_gbs", "16"),
216+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_dataflow_configuration_details.0.driver_shape_config_details.0.ocpus", "1"),
217+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_dataflow_configuration_details.0.executor_shape", "VM.Standard.E5.Flex"),
218+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_dataflow_configuration_details.0.executor_shape_config_details.#", "1"),
219+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_dataflow_configuration_details.0.executor_shape_config_details.0.memory_in_gbs", "16"),
220+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_dataflow_configuration_details.0.executor_shape_config_details.0.ocpus", "1"),
221+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_dataflow_configuration_details.0.logs_bucket_uri", "oci://xuejuzha-test@idtlxnfdweil/"),
222+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_dataflow_configuration_details.0.num_executors", "1"),
223+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_dataflow_configuration_details.0.warehouse_bucket_uri", "oci://xuejuzha-test@idtlxnfdweil/"),
224+
225+
func(s *terraform.State) (err error) {
226+
resId, err = acctest.FromInstanceState(s, resourceName, "id")
227+
return err
228+
},
229+
),
230+
},
231+
{
232+
Config: config + compartmentIdVariableStr + PipelineRunContainerResourceDependencies, // current pipeline state = ACCEPTED and DELETE after SUCCEEDED/CANCELED/FAILED
233+
},
147234
{
148235
Config: config + compartmentIdVariableStr + PipelineRunContainerResourceDependencies +
149236
acctest.GenerateResourceFromRepresentationMap("oci_datascience_pipeline_run", "test_pipeline_run", acctest.Optional, acctest.Create, pipelineRunContainerRepresentation),
@@ -161,9 +248,9 @@ func TestDatasciencePipelineRunResource_basic(t *testing.T) {
161248
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_name", "stepNameContainer"),
162249
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_container_configuration_details.#", "1"),
163250
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_container_configuration_details.0.container_type", "OCIR_CONTAINER"),
164-
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_container_configuration_details.0.image", "iad.ocir.io/ociodscdev/nested-rp-public-python-sdk-1:1.0"),
165-
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_container_configuration_details.0.cmd.#", "1"),
166-
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_container_configuration_details.0.entrypoint.#", "1"),
251+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_container_configuration_details.0.image", "iad.ocir.io/idtlxnfdweil/byod-hello-world:1"),
252+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_container_configuration_details.0.cmd.#", "0"),
253+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_container_configuration_details.0.entrypoint.#", "0"),
167254

168255
func(s *terraform.State) (err error) {
169256
resId, err = acctest.FromInstanceState(s, resourceName, "id")

0 commit comments

Comments
 (0)