Skip to content

Commit 5176eab

Browse files
Shreya PatelJulia3072
Shreya Patel
authored and
Julia3072
committed
Added - Support for ODSC - Pipelines v2
1 parent 9610f0f commit 5176eab

11 files changed

+1070
-242
lines changed

‎examples/datascience/pipeline/pipeline.tf‎

Lines changed: 103 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,10 @@ variable "pipeline_step_details_description" {
9292
}
9393

9494
variable "pipeline_step_details_is_artifact_uploaded" {
95+
default = true
96+
}
97+
98+
variable "pipeline_container_step_details_is_artifact_uploaded" {
9599
default = false
96100
}
97101

@@ -129,7 +133,9 @@ variable "pipeline_step_details_step_type" {
129133
default = "CUSTOM_SCRIPT"
130134
}
131135

132-
136+
variable "pipeline_step_details_step_type_container" {
137+
default = "CONTAINER"
138+
}
133139

134140
provider "oci" {
135141
tenancy_ocid = var.tenancy_ocid
@@ -230,6 +236,65 @@ resource "oci_datascience_pipeline" "test_pipeline" {
230236
}
231237
}
232238

239+
resource "oci_datascience_pipeline" "test_pipeline_byoc" {
240+
#Required
241+
compartment_id = var.compartment_ocid
242+
project_id = oci_datascience_project.pipeline.id
243+
delete_related_pipeline_runs = true
244+
step_details {
245+
#Required
246+
step_name = var.pipeline_step_details_step_name
247+
step_type = var.pipeline_step_details_step_type_container
248+
249+
#Optional
250+
depends_on = var.pipeline_step_details_depends_on
251+
description = var.pipeline_step_details_description
252+
is_artifact_uploaded = var.pipeline_step_details_is_artifact_uploaded
253+
254+
step_configuration_details {
255+
#Optional
256+
command_line_arguments = var.pipeline_step_details_step_configuration_details_command_line_arguments
257+
environment_variables = var.pipeline_step_details_step_configuration_details_environment_variables
258+
maximum_runtime_in_minutes = var.pipeline_step_details_step_configuration_details_maximum_runtime_in_minutes
259+
}
260+
step_infrastructure_configuration_details {
261+
#Optional
262+
block_storage_size_in_gbs = var.pipeline_step_details_step_infrastructure_configuration_details_block_storage_size_in_gbs
263+
shape_name = "VM.Standard2.1"
264+
}
265+
step_container_configuration_details {
266+
#Required
267+
cmd = ["hello_world.py"]
268+
container_type = "OCIR_CONTAINER"
269+
entrypoint = ["python3"]
270+
image = "iad.ocir.io/ociodscdev/nested-rp-public-python-sdk-1:1.0"
271+
#Optional
272+
image_digest = ""
273+
image_signature_id = ""
274+
}
275+
}
276+
277+
#Optional
278+
configuration_details {
279+
#Required
280+
type = var.pipeline_configuration_details_type
281+
282+
#Optional
283+
command_line_arguments = var.pipeline_configuration_details_command_line_arguments
284+
environment_variables = var.pipeline_configuration_details_environment_variables
285+
maximum_runtime_in_minutes = var.pipeline_configuration_details_maximum_runtime_in_minutes
286+
}
287+
description = var.pipeline_description
288+
display_name = var.pipeline_display_name
289+
290+
infrastructure_configuration_details {
291+
#Required
292+
block_storage_size_in_gbs = var.pipeline_infrastructure_configuration_details_block_storage_size_in_gbs
293+
shape_name = "VM.Standard2.1"
294+
}
295+
}
296+
297+
233298
resource "oci_logging_log_group" "pipeline_run" {
234299
compartment_id = var.compartment_ocid
235300
display_name = "pipeline_run"
@@ -252,6 +317,7 @@ resource "oci_datascience_pipeline_run" "test_pipeline_run" {
252317
maximum_runtime_in_minutes = 30
253318
}
254319
display_name = "DisplayName1"
320+
#Optional
255321
log_configuration_override_details {
256322

257323
#Optional
@@ -272,4 +338,40 @@ resource "oci_datascience_pipeline_run" "test_pipeline_run" {
272338
}
273339
step_name = "stepName"
274340
}
341+
}
342+
343+
resource "oci_datascience_pipeline_run" "test_pipeline_run_byoc" {
344+
#Required
345+
compartment_id = var.compartment_ocid
346+
pipeline_id = oci_datascience_pipeline.test_pipeline_byoc.id
347+
delete_related_job_runs = true
348+
349+
display_name = "DisplayNameContainer"
350+
#Optional
351+
log_configuration_override_details {
352+
353+
#Optional
354+
enable_auto_log_creation = false
355+
enable_logging = false
356+
// log_group_id = oci_logging_log_group.pipeline_run.id
357+
# log_id = oci_logging_log.test_log.id
358+
}
359+
project_id = oci_datascience_project.pipeline.id
360+
step_override_details {
361+
#Required
362+
step_configuration_details {
363+
364+
#Optional
365+
command_line_arguments = "CommandLineArgumentsStepOverride"
366+
environment_variables = {"environmentVariablesStepOverride":"environmentVariablesStepOverride"}
367+
maximum_runtime_in_minutes = 30
368+
}
369+
step_name = "stepName"
370+
step_container_configuration_details {
371+
container_type = "OCIR_CONTAINER"
372+
image = "iad.ocir.io/ociodscdev/nested-rp-public-python-sdk-1:1.0"
373+
cmd = ["hello_world.py"]
374+
entrypoint = ["python3"]
375+
}
376+
}
275377
}

‎internal/integrationtest/datascience_pipeline_run_test.go‎

Lines changed: 76 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package integrationtest
66
import (
77
"context"
88
"fmt"
9+
"github.com/oracle/terraform-provider-oci/internal/resourcediscovery"
910
"strconv"
1011
"testing"
1112
"time"
@@ -18,7 +19,6 @@ import (
1819
"github.com/oracle/terraform-provider-oci/httpreplay"
1920
acctest "github.com/oracle/terraform-provider-oci/internal/acctest"
2021
tf_client "github.com/oracle/terraform-provider-oci/internal/client"
21-
"github.com/oracle/terraform-provider-oci/internal/resourcediscovery"
2222
"github.com/oracle/terraform-provider-oci/internal/tfresource"
2323
"github.com/oracle/terraform-provider-oci/internal/utils"
2424
)
@@ -36,7 +36,7 @@ var (
3636

3737
pipelineRunDataSourceRepresentation = map[string]interface{}{
3838
"compartment_id": acctest.Representation{RepType: acctest.Required, Create: `${var.compartment_id}`},
39-
"created_by": acctest.Representation{RepType: acctest.Optional, Create: `createdBy`},
39+
"created_by": acctest.Representation{RepType: acctest.Optional, Create: `${oci_datascience_pipeline.test_pipeline.created_by}`},
4040
"display_name": acctest.Representation{RepType: acctest.Optional, Create: `displayName`, Update: `displayName2`},
4141
"id": acctest.Representation{RepType: acctest.Optional, Create: `${oci_datascience_pipeline_run.test_pipeline_run.id}`},
4242
"pipeline_id": acctest.Representation{RepType: acctest.Optional, Create: `${oci_datascience_pipeline.test_pipeline.id}`},
@@ -59,6 +59,19 @@ var (
5959
"log_configuration_override_details": acctest.RepresentationGroup{RepType: acctest.Optional, Group: pipelineRunLogConfigurationOverrideDetailsRepresentation},
6060
"step_override_details": acctest.RepresentationGroup{RepType: acctest.Optional, Group: pipelineRunStepOverrideDetailsRepresentation},
6161
}
62+
63+
pipelineRunContainerRepresentation = map[string]interface{}{
64+
"compartment_id": acctest.Representation{RepType: acctest.Required, Create: `${var.compartment_id}`},
65+
"pipeline_id": acctest.Representation{RepType: acctest.Required, Create: `${oci_datascience_pipeline.test_pipeline.id}`},
66+
"project_id": acctest.Representation{RepType: acctest.Required, Create: `${oci_datascience_project.test_project.id}`},
67+
"configuration_override_details": acctest.RepresentationGroup{RepType: acctest.Optional, Group: pipelineRunConfigurationOverrideDetailsRepresentation},
68+
"display_name": acctest.Representation{RepType: acctest.Optional, Create: `displayName`, Update: `displayName2`},
69+
"freeform_tags": acctest.Representation{RepType: acctest.Optional, Create: map[string]string{"Department": "Finance"}, Update: map[string]string{"Department": "Accounting"}},
70+
"delete_related_job_runs": acctest.Representation{RepType: acctest.Required, Create: `true`, Update: `true`},
71+
"log_configuration_override_details": acctest.RepresentationGroup{RepType: acctest.Optional, Group: pipelineRunLogConfigurationOverrideDetailsRepresentation},
72+
"step_override_details": acctest.RepresentationGroup{RepType: acctest.Optional, Group: pipelineRunStepOverrideDetailsContainerRepresentation},
73+
}
74+
6275
pipelineRunConfigurationOverrideDetailsRepresentation = map[string]interface{}{
6376
"type": acctest.Representation{RepType: acctest.Required, Create: `DEFAULT`},
6477
"command_line_arguments": acctest.Representation{RepType: acctest.Optional, Create: `commandLineArgumentsOverriden`},
@@ -74,17 +87,36 @@ var (
7487
pipelineRunStepOverrideDetailsRepresentation = map[string]interface{}{
7588
"step_configuration_details": acctest.RepresentationGroup{RepType: acctest.Required, Group: pipelineRunStepOverrideDetailsStepConfigurationDetailsRepresentation},
7689
"step_name": acctest.Representation{RepType: acctest.Required, Create: `stepName`},
90+
//"step_container_configuration_details": acctest.RepresentationGroup{RepType: acctest.Optional, Group: DatasciencePipelineRunStepOverrideDetailsContainerConfigurationDetailsRepresentation},
91+
}
92+
pipelineRunStepOverrideDetailsContainerRepresentation = map[string]interface{}{
93+
"step_configuration_details": acctest.RepresentationGroup{RepType: acctest.Required, Group: pipelineRunStepOverrideDetailsStepConfigurationDetailsRepresentation},
94+
"step_name": acctest.Representation{RepType: acctest.Required, Create: `stepNameContainer`},
95+
"step_container_configuration_details": acctest.RepresentationGroup{RepType: acctest.Optional, Group: DatasciencePipelineRunStepOverrideDetailsContainerConfigurationDetailsRepresentation},
7796
}
7897
pipelineRunStepOverrideDetailsStepConfigurationDetailsRepresentation = map[string]interface{}{
7998
"command_line_arguments": acctest.Representation{RepType: acctest.Optional, Create: `commandLineArgumentsOverriden`},
8099
"environment_variables": acctest.Representation{RepType: acctest.Optional, Create: map[string]string{"environmentVariables": "environmentVariablesOverriden"}},
81100
"maximum_runtime_in_minutes": acctest.Representation{RepType: acctest.Optional, Create: `10`},
82101
}
102+
DatasciencePipelineRunStepOverrideDetailsContainerConfigurationDetailsRepresentation = map[string]interface{}{
103+
"container_type": acctest.Representation{RepType: acctest.Required, Create: `OCIR_CONTAINER`},
104+
"image": acctest.Representation{RepType: acctest.Required, Create: `iad.ocir.io/ociodscdev/nested-rp-public-python-sdk-1:1.0`},
105+
"cmd": acctest.Representation{RepType: acctest.Optional, Create: []string{`hello_world.py`}},
106+
"entrypoint": acctest.Representation{RepType: acctest.Optional, Create: []string{`python3`}},
107+
"image_digest": acctest.Representation{RepType: acctest.Optional, Create: ``},
108+
"image_signature_id": acctest.Representation{RepType: acctest.Optional, Create: ``},
109+
}
83110

84111
PipelineRunResourceDependencies = acctest.GenerateResourceFromRepresentationMap("oci_datascience_pipeline", "test_pipeline", acctest.Required, acctest.Create, pipelineRepresentation) +
85112
acctest.GenerateResourceFromRepresentationMap("oci_datascience_project", "test_project", acctest.Required, acctest.Create, DatascienceProjectRepresentation) +
86113
acctest.GenerateResourceFromRepresentationMap("oci_logging_log_group", "terraform_test_custom_log_group", acctest.Required, acctest.Create, pipelineLogGroupRepresentation) +
87114
acctest.GenerateResourceFromRepresentationMap("oci_logging_log", "terraform_test_custom_log", acctest.Required, acctest.Create, pipelineLogRepresentation)
115+
116+
PipelineRunContainerResourceDependencies = acctest.GenerateResourceFromRepresentationMap("oci_datascience_pipeline", "test_pipeline", acctest.Optional, acctest.Create, pipelineRepresentationContainer) +
117+
acctest.GenerateResourceFromRepresentationMap("oci_datascience_project", "test_project", acctest.Required, acctest.Create, DatascienceProjectRepresentation) +
118+
acctest.GenerateResourceFromRepresentationMap("oci_logging_log_group", "terraform_test_custom_log_group", acctest.Required, acctest.Create, pipelineLogGroupRepresentation) +
119+
acctest.GenerateResourceFromRepresentationMap("oci_logging_log", "terraform_test_custom_log", acctest.Required, acctest.Create, pipelineLogRepresentation)
88120
)
89121

90122
// issue-routing-tag: datascience/default
@@ -106,11 +138,43 @@ func TestDatasciencePipelineRunResource_basic(t *testing.T) {
106138

107139
var resId, resId2 string
108140
// Save TF content to Create resource with optional properties. This has to be exactly the same as the config part in the "create with optionals" step in the test.
109-
acctest.SaveConfigContent(config+compartmentIdVariableStr+PipelineRunResourceDependencies+
141+
acctest.SaveConfigContent(config+compartmentIdVariableStr+PipelineRunContainerResourceDependencies+
110142
acctest.GenerateResourceFromRepresentationMap("oci_datascience_pipeline_run", "test_pipeline_run", acctest.Optional, acctest.Create, pipelineRunRepresentation), "datascience", "pipelineRun", t)
111143

112144
acctest.ResourceTest(t, testAccCheckDatasciencePipelineRunDestroy, []resource.TestStep{
113-
// verify Create
145+
// Step 0 - Verify Create Pipeline Run with Container
146+
{
147+
Config: config + compartmentIdVariableStr + PipelineRunContainerResourceDependencies +
148+
acctest.GenerateResourceFromRepresentationMap("oci_datascience_pipeline_run", "test_pipeline_run", acctest.Optional, acctest.Create, pipelineRunContainerRepresentation),
149+
Check: acctest.ComposeAggregateTestCheckFuncWrapper(
150+
resource.TestCheckResourceAttr(resourceName, "compartment_id", compartmentId),
151+
resource.TestCheckResourceAttrSet(resourceName, "pipeline_id"),
152+
resource.TestCheckResourceAttrSet(resourceName, "project_id"),
153+
resource.TestCheckResourceAttrSet(resourceName, "id"),
154+
resource.TestCheckResourceAttr(resourceName, "configuration_override_details.#", "1"),
155+
resource.TestCheckResourceAttr(resourceName, "step_override_details.#", "1"),
156+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_configuration_details.#", "1"),
157+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_configuration_details.0.command_line_arguments", "commandLineArgumentsOverriden"),
158+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_configuration_details.0.environment_variables.%", "1"),
159+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_configuration_details.0.maximum_runtime_in_minutes", "10"),
160+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_name", "stepNameContainer"),
161+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_container_configuration_details.#", "1"),
162+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_container_configuration_details.0.container_type", "OCIR_CONTAINER"),
163+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_container_configuration_details.0.image", "iad.ocir.io/ociodscdev/nested-rp-public-python-sdk-1:1.0"),
164+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_container_configuration_details.0.cmd.#", "1"),
165+
resource.TestCheckResourceAttr(resourceName, "step_override_details.0.step_container_configuration_details.0.entrypoint.#", "1"),
166+
167+
func(s *terraform.State) (err error) {
168+
resId, err = acctest.FromInstanceState(s, resourceName, "id")
169+
return err
170+
},
171+
),
172+
},
173+
// Step 1 - delete before next Create
174+
{
175+
Config: config + compartmentIdVariableStr + PipelineRunContainerResourceDependencies, // current pipeline state = ACCEPTED and DELETE after SUCCEEDED/CANCELED/FAILED
176+
},
177+
// Step 2 - verify Create
114178
{
115179
Config: config + compartmentIdVariableStr + PipelineRunResourceDependencies +
116180
acctest.GenerateResourceFromRepresentationMap("oci_datascience_pipeline_run", "test_pipeline_run", acctest.Required, acctest.Create, pipelineRunRepresentation),
@@ -125,12 +189,11 @@ func TestDatasciencePipelineRunResource_basic(t *testing.T) {
125189
},
126190
),
127191
},
128-
129-
// delete before next Create
192+
// Step 3 - delete before next Create
130193
{
131194
Config: config + compartmentIdVariableStr + PipelineRunResourceDependencies, // current pipeline state = ACCEPTED and DELETE after SUCCEEDED/CANCELED/FAILED
132195
},
133-
// verify Create with optionals
196+
// Step 4 - verify Create with optionals
134197
{
135198
Config: config + compartmentIdVariableStr + PipelineRunResourceDependencies +
136199
acctest.GenerateResourceFromRepresentationMap("oci_datascience_pipeline_run", "test_pipeline_run", acctest.Optional, acctest.Create, pipelineRunRepresentation),
@@ -175,7 +238,7 @@ func TestDatasciencePipelineRunResource_basic(t *testing.T) {
175238
),
176239
},
177240

178-
// verify Update to the compartment (the compartment will be switched back in the next step)
241+
// Step 5 - verify Update to the compartment (the compartment will be switched back in the next step)
179242
{
180243
Config: config + compartmentIdVariableStr + compartmentIdUVariableStr + PipelineRunResourceDependencies +
181244
acctest.GenerateResourceFromRepresentationMap("oci_datascience_pipeline_run", "test_pipeline_run", acctest.Optional, acctest.Create,
@@ -221,7 +284,7 @@ func TestDatasciencePipelineRunResource_basic(t *testing.T) {
221284
),
222285
},
223286

224-
// verify updates to updatable parameters
287+
// Step 6 - verify updates to updatable parameters
225288
{
226289
Config: config + compartmentIdVariableStr + PipelineRunResourceDependencies +
227290
acctest.GenerateResourceFromRepresentationMap("oci_datascience_pipeline_run", "test_pipeline_run", acctest.Optional, acctest.Update, pipelineRunRepresentation),
@@ -263,20 +326,19 @@ func TestDatasciencePipelineRunResource_basic(t *testing.T) {
263326
},
264327
),
265328
},
266-
// verify datasource
329+
// Step 7 - verify datasource
267330
{
268331
Config: config +
269332
acctest.GenerateDataSourceFromRepresentationMap("oci_datascience_pipeline_runs", "test_pipeline_runs", acctest.Optional, acctest.Update, pipelineRunDataSourceRepresentation) +
270333
compartmentIdVariableStr + PipelineRunResourceDependencies +
271334
acctest.GenerateResourceFromRepresentationMap("oci_datascience_pipeline_run", "test_pipeline_run", acctest.Optional, acctest.Update, pipelineRunRepresentation),
272335
Check: acctest.ComposeAggregateTestCheckFuncWrapper(
273336
resource.TestCheckResourceAttr(datasourceName, "compartment_id", compartmentId),
274-
resource.TestCheckResourceAttr(datasourceName, "created_by", "createdBy"),
337+
resource.TestCheckResourceAttrSet(datasourceName, "created_by"),
275338
resource.TestCheckResourceAttr(datasourceName, "display_name", "displayName2"),
276339
resource.TestCheckResourceAttrSet(datasourceName, "id"),
277340
resource.TestCheckResourceAttrSet(datasourceName, "pipeline_id"),
278341
resource.TestCheckResourceAttr(datasourceName, "state", "SUCCEEDED"),
279-
280342
resource.TestCheckResourceAttr(datasourceName, "pipeline_runs.#", "1"),
281343
resource.TestCheckResourceAttr(datasourceName, "pipeline_runs.0.compartment_id", compartmentId),
282344
resource.TestCheckResourceAttrSet(datasourceName, "pipeline_runs.0.created_by"),
@@ -292,14 +354,13 @@ func TestDatasciencePipelineRunResource_basic(t *testing.T) {
292354
resource.TestCheckResourceAttrSet(datasourceName, "pipeline_runs.0.time_started"),
293355
),
294356
},
295-
// verify singular datasource
357+
// Step 8 - verify singular datasource
296358
{
297359
Config: config +
298360
acctest.GenerateDataSourceFromRepresentationMap("oci_datascience_pipeline_run", "test_pipeline_run", acctest.Required, acctest.Create, pipelineRunSingularDataSourceRepresentation) +
299361
compartmentIdVariableStr + PipelineRunResourceConfig,
300362
Check: acctest.ComposeAggregateTestCheckFuncWrapper(
301363
resource.TestCheckResourceAttrSet(singularDatasourceName, "pipeline_run_id"),
302-
303364
resource.TestCheckResourceAttr(singularDatasourceName, "compartment_id", compartmentId),
304365
resource.TestCheckResourceAttr(singularDatasourceName, "configuration_override_details.#", "1"),
305366
resource.TestCheckResourceAttr(singularDatasourceName, "configuration_override_details.0.command_line_arguments", "commandLineArgumentsOverriden"),
@@ -328,7 +389,7 @@ func TestDatasciencePipelineRunResource_basic(t *testing.T) {
328389
resource.TestCheckResourceAttrSet(singularDatasourceName, "time_started"),
329390
),
330391
},
331-
// verify resource import
392+
// Step 9 - verify resource import
332393
{
333394
Config: config + PipelineRunRequiredOnlyResource,
334395
ImportState: true,

0 commit comments

Comments
 (0)