Skip to content

Commit f26b642

Browse files
Terraform Team Automationsagarp337
authored andcommitted
Added - Support for ODSC - ML Pipelines
1 parent 9e443f3 commit f26b642

24 files changed

+5348
-25
lines changed
6.79 KB
Binary file not shown.
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# https://orahub.oci.oraclecorp.com/ateam/jobs/-/raw/master/python_jobs/job_std_logs.py
2+
3+
import oci
4+
import datetime
5+
import os
6+
import time
7+
import sys
8+
from oci.loggingingestion import LoggingClient
9+
10+
OCI_RESOURCE_PRINCIPAL_VERSION = "OCI_RESOURCE_PRINCIPAL_VERSION"
11+
JOB_RUN_OCID_KEY = "JOB_RUN_OCID"
12+
LOG_OBJECT_OCID_KEY = "LOG_OCID"
13+
14+
class Job:
15+
def __init__(self):
16+
# Auto switch between local and job
17+
rp_version = os.environ.get(OCI_RESOURCE_PRINCIPAL_VERSION, "UNDEFINED")
18+
if rp_version == "UNDEFINED":
19+
# RUN LOCAL TEST
20+
self.signer = oci.config.from_file("~/.oci/config", "DEFAULT")
21+
self.log_client = LoggingClient(config=self.signer)
22+
else:
23+
# RUN AS JOB
24+
self.signer = oci.auth.signers.get_resource_principals_signer()
25+
self.log_client = LoggingClient(config={}, signer=self.signer)
26+
try:
27+
job = Job()
28+
29+
try:
30+
print(
31+
"Start logging for job run: {}".format(
32+
os.environ.get(JOB_RUN_OCID_KEY, "UNDEFINED")
33+
)
34+
)
35+
print("Current timestamp in UTC: {}".format(str(datetime.datetime.utcnow())))
36+
37+
print("Delay 5s")
38+
39+
time.sleep(5)
40+
41+
print("Log after delay...")
42+
43+
print("Job Done.")
44+
except Exception as e:
45+
print(e)
46+
raise e
47+
except Exception as e:
48+
print(e)
49+
raise e
Lines changed: 275 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,275 @@
1+
// Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
2+
// Licensed under the Mozilla Public License v2.0
3+
4+
variable "tenancy_ocid" {
5+
6+
}
7+
variable "user_ocid" {
8+
9+
}
10+
variable "fingerprint" {
11+
12+
}
13+
variable "private_key_path" {
14+
15+
}
16+
variable "region" {
17+
default = "us-ashburn-1"
18+
}
19+
variable "compartment_ocid" {
20+
default = "ocid1.tenancy.oc1..aaaaaaaahzy3x4boh7ipxyft2rowu2xeglvanlfewudbnueugsieyuojkldq"
21+
}
22+
23+
variable "pipeline_configuration_details_command_line_arguments" {
24+
default ="commandLineArguments"
25+
26+
}
27+
28+
variable "pipeline_configuration_details_environment_variables" {
29+
default = {
30+
"environmentVariables": "environmentVariables"
31+
}
32+
}
33+
34+
variable "pipeline_configuration_details_maximum_runtime_in_minutes" {
35+
default = 20
36+
}
37+
38+
variable "pipeline_configuration_details_type" {
39+
default = "DEFAULT"
40+
}
41+
42+
variable "pipeline_defined_tags_value" {
43+
default = "value"
44+
}
45+
46+
variable "pipeline_description" {
47+
default = "description"
48+
}
49+
50+
variable "pipeline_display_name" {
51+
default = "displayName"
52+
}
53+
54+
variable "pipeline_freeform_tags" {
55+
default = { "Department" = "Finance" }
56+
}
57+
58+
variable "pipeline_id" {
59+
default = "id"
60+
}
61+
62+
variable "pipeline_infrastructure_configuration_details_block_storage_size_in_gbs" {
63+
default = 50
64+
}
65+
66+
variable "pipeline_infrastructure_configuration_details_shape_config_details_memory_in_gbs" {
67+
default = 1.0
68+
}
69+
70+
variable "pipeline_infrastructure_configuration_details_shape_config_details_ocpus" {
71+
default = 1.0
72+
}
73+
74+
variable "pipeline_log_configuration_details_enable_auto_log_creation" {
75+
default = true
76+
}
77+
78+
variable "pipeline_log_configuration_details_enable_logging" {
79+
default = true
80+
}
81+
82+
variable "pipeline_state" {
83+
default = "AVAILABLE"
84+
}
85+
86+
variable "pipeline_step_details_depends_on" {
87+
default = []
88+
}
89+
90+
variable "pipeline_step_details_description" {
91+
default = "description"
92+
}
93+
94+
variable "pipeline_step_details_is_artifact_uploaded" {
95+
default = false
96+
}
97+
98+
variable "pipeline_step_details_step_configuration_details_command_line_arguments" {
99+
default = "commandLineArguments"
100+
}
101+
102+
variable "pipeline_step_details_step_configuration_details_environment_variables" {
103+
default ={
104+
"environmentVariables": "environmentVariables"
105+
}
106+
}
107+
108+
variable "pipeline_step_details_step_configuration_details_maximum_runtime_in_minutes" {
109+
default = 10
110+
}
111+
112+
variable "pipeline_step_details_step_infrastructure_configuration_details_block_storage_size_in_gbs" {
113+
default = 50
114+
}
115+
116+
variable "pipeline_step_details_step_infrastructure_configuration_details_shape_config_details_memory_in_gbs" {
117+
default = 1.0
118+
}
119+
120+
variable "pipeline_step_details_step_infrastructure_configuration_details_shape_config_details_ocpus" {
121+
default = 1.0
122+
}
123+
124+
variable "pipeline_step_details_step_name" {
125+
default = "stepName"
126+
}
127+
128+
variable "pipeline_step_details_step_type" {
129+
default = "CUSTOM_SCRIPT"
130+
}
131+
132+
133+
134+
provider "oci" {
135+
tenancy_ocid = var.tenancy_ocid
136+
user_ocid = var.user_ocid
137+
fingerprint = var.fingerprint
138+
private_key_path = var.private_key_path
139+
region = var.region
140+
}
141+
142+
resource "oci_datascience_project" "pipeline" {
143+
compartment_id = var.compartment_ocid
144+
}
145+
146+
resource "oci_logging_log_group" "pipeline" {
147+
compartment_id = var.compartment_ocid
148+
display_name = "pipelines"
149+
}
150+
151+
152+
resource "oci_datascience_pipeline" "test_pipeline" {
153+
#Required
154+
compartment_id = var.compartment_ocid
155+
project_id = oci_datascience_project.pipeline.id
156+
delete_related_pipeline_runs = true
157+
step_details {
158+
#Required
159+
step_name = var.pipeline_step_details_step_name
160+
step_type = var.pipeline_step_details_step_type
161+
162+
#Optional
163+
depends_on = var.pipeline_step_details_depends_on
164+
description = var.pipeline_step_details_description
165+
is_artifact_uploaded = var.pipeline_step_details_is_artifact_uploaded
166+
# No Job Id for CUSTOM_SCRIPT
167+
# job_id = oci_datascience_job.test_job.id
168+
step_configuration_details {
169+
170+
#Optional
171+
command_line_arguments = var.pipeline_step_details_step_configuration_details_command_line_arguments
172+
environment_variables = var.pipeline_step_details_step_configuration_details_environment_variables
173+
maximum_runtime_in_minutes = var.pipeline_step_details_step_configuration_details_maximum_runtime_in_minutes
174+
}
175+
step_infrastructure_configuration_details {
176+
177+
#Optional
178+
block_storage_size_in_gbs = var.pipeline_step_details_step_infrastructure_configuration_details_block_storage_size_in_gbs
179+
# shape_config_details {
180+
181+
# #Optional
182+
# memory_in_gbs = var.pipeline_step_details_step_infrastructure_configuration_details_shape_config_details_memory_in_gbs
183+
# ocpus = var.pipeline_step_details_step_infrastructure_configuration_details_shape_config_details_ocpus
184+
# }
185+
shape_name = "VM.Standard2.1"
186+
}
187+
}
188+
189+
#Optional in ML_JOB step type, mandatory for CUSTOM_SCRIPT step type
190+
step_artifact {
191+
step_name = "stepName"
192+
pipeline_step_artifact = "${path.root}/pipeline-artifact.py"
193+
artifact_content_length = 1380
194+
artifact_content_disposition = "attachment; filename=pipeline_artifact.py"
195+
}
196+
197+
#Optional
198+
configuration_details {
199+
#Required
200+
type = var.pipeline_configuration_details_type
201+
202+
#Optional
203+
command_line_arguments = var.pipeline_configuration_details_command_line_arguments
204+
environment_variables = var.pipeline_configuration_details_environment_variables
205+
maximum_runtime_in_minutes = var.pipeline_configuration_details_maximum_runtime_in_minutes
206+
}
207+
# defined_tags = map(oci_identity_tag_namespace.tag-namespace1.name.oci_identity_tag.tag1.name, var.pipeline_defined_tags_value)
208+
description = var.pipeline_description
209+
display_name = var.pipeline_display_name
210+
# freeform_tags = var.pipeline_freeform_tags
211+
infrastructure_configuration_details {
212+
#Required
213+
block_storage_size_in_gbs = var.pipeline_infrastructure_configuration_details_block_storage_size_in_gbs
214+
shape_name = "VM.Standard2.1"
215+
#Optional ONLY required if the shape is a flex shape
216+
# shape_config_details {
217+
218+
# #Optional
219+
# memory_in_gbs = var.pipeline_infrastructure_configuration_details_shape_config_details_memory_in_gbs
220+
# ocpus = var.pipeline_infrastructure_configuration_details_shape_config_details_ocpus
221+
# }
222+
}
223+
log_configuration_details {
224+
225+
#Optional
226+
enable_auto_log_creation = var.pipeline_log_configuration_details_enable_auto_log_creation
227+
enable_logging = var.pipeline_log_configuration_details_enable_logging
228+
log_group_id = oci_logging_log_group.pipeline.id
229+
# log_id = oci_logging_log.test_log.id
230+
}
231+
}
232+
233+
resource "oci_logging_log_group" "pipeline_run" {
234+
compartment_id = var.compartment_ocid
235+
display_name = "pipeline_run"
236+
}
237+
238+
resource "oci_datascience_pipeline_run" "test_pipeline_run" {
239+
#Required
240+
compartment_id = var.compartment_ocid
241+
pipeline_id = oci_datascience_pipeline.test_pipeline.id
242+
delete_related_job_runs = true
243+
244+
#Optional
245+
configuration_override_details {
246+
#Required
247+
type = "DEFAULT"
248+
249+
#Optional
250+
command_line_arguments = "CommandLineArgumentsOverride"
251+
environment_variables = {"environmentVariablesOverride":"environmentVariablesOverride"}
252+
maximum_runtime_in_minutes = 30
253+
}
254+
display_name = "DisplayName1"
255+
log_configuration_override_details {
256+
257+
#Optional
258+
enable_auto_log_creation = true
259+
enable_logging = true
260+
log_group_id = oci_logging_log_group.pipeline_run.id
261+
# log_id = oci_logging_log.test_log.id
262+
}
263+
project_id = oci_datascience_project.pipeline.id
264+
step_override_details {
265+
#Required
266+
step_configuration_details {
267+
268+
#Optional
269+
command_line_arguments = "CommandLineArgumentsStepOverride"
270+
environment_variables = {"environmentVariablesStepOverride":"environmentVariablesStepOverride"}
271+
maximum_runtime_in_minutes = 30
272+
}
273+
step_name = "stepName"
274+
}
275+
}

internal/integrationtest/datascience_model_test.go

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -37,14 +37,12 @@ var (
3737
}
3838

3939
DatascienceDatascienceModelDataSourceRepresentation = map[string]interface{}{
40-
"compartment_id": acctest.Representation{RepType: acctest.Required, Create: `${var.compartment_id}`},
41-
"display_name": acctest.Representation{RepType: acctest.Optional, Create: `displayName`, Update: `displayName2`},
42-
"id": acctest.Representation{RepType: acctest.Optional, Create: `${oci_datascience_model.test_model.id}`},
43-
"model_version_set_name": acctest.Representation{RepType: acctest.Optional, Create: ``},
44-
"project_id": acctest.Representation{RepType: acctest.Optional, Create: `${oci_datascience_project.test_project.id}`},
45-
"state": acctest.Representation{RepType: acctest.Optional, Create: `ACTIVE`},
46-
"version_label": acctest.Representation{RepType: acctest.Optional, Create: ``, Update: ``},
47-
"filter": acctest.RepresentationGroup{RepType: acctest.Required, Group: DatascienceModelDataSourceFilterRepresentation}}
40+
"compartment_id": acctest.Representation{RepType: acctest.Required, Create: `${var.compartment_id}`},
41+
"display_name": acctest.Representation{RepType: acctest.Optional, Create: `displayName`, Update: `displayName2`},
42+
"id": acctest.Representation{RepType: acctest.Optional, Create: `${oci_datascience_model.test_model.id}`},
43+
"project_id": acctest.Representation{RepType: acctest.Optional, Create: `${oci_datascience_project.test_project.id}`},
44+
"state": acctest.Representation{RepType: acctest.Optional, Create: `ACTIVE`},
45+
"filter": acctest.RepresentationGroup{RepType: acctest.Required, Group: DatascienceModelDataSourceFilterRepresentation}}
4846
DatascienceModelDataSourceFilterRepresentation = map[string]interface{}{
4947
"name": acctest.Representation{RepType: acctest.Required, Create: `id`},
5048
"values": acctest.Representation{RepType: acctest.Required, Create: []string{`${oci_datascience_model.test_model.id}`}},
@@ -78,10 +76,6 @@ var (
7876

7977
DatascienceModelResourceDependencies = acctest.GenerateResourceFromRepresentationMap("oci_datascience_project", "test_project", acctest.Required, acctest.Create, DatascienceProjectRepresentation) +
8078
DefinedTagsDependencies
81-
82-
DatascienceModelResourceModelVersionSetDependencies = acctest.GenerateResourceFromRepresentationMap("oci_datascience_model_version_set", "test_model_version_set", acctest.Required, acctest.Create, DatascienceModelVersionSetRepresentation) +
83-
acctest.GenerateResourceFromRepresentationMap("oci_datascience_project", "test_project", acctest.Required, acctest.Create, DatascienceProjectRepresentation) +
84-
DefinedTagsDependencies
8579
)
8680

8781
// issue-routing-tag: datascience/default
@@ -247,12 +241,11 @@ func TestDatascienceModelResource_basic(t *testing.T) {
247241
{
248242
Config: config +
249243
acctest.GenerateDataSourceFromRepresentationMap("oci_datascience_models", "test_models", acctest.Optional, acctest.Update, DatascienceDatascienceModelDataSourceRepresentation) +
250-
compartmentIdVariableStr + DatascienceModelResourceModelVersionSetDependencies +
244+
compartmentIdVariableStr + DatascienceModelResourceDependencies +
251245
acctest.GenerateResourceFromRepresentationMap("oci_datascience_model", "test_model", acctest.Optional, acctest.Update, DatascienceModelRepresentation),
252246
Check: acctest.ComposeAggregateTestCheckFuncWrapper(
253247
resource.TestCheckResourceAttr(datasourceName, "compartment_id", compartmentId),
254248
resource.TestCheckResourceAttr(datasourceName, "display_name", "displayName2"),
255-
resource.TestCheckResourceAttrSet(datasourceName, "id"),
256249
resource.TestCheckResourceAttrSet(datasourceName, "project_id"),
257250
resource.TestCheckResourceAttr(datasourceName, "state", "ACTIVE"),
258251

0 commit comments

Comments
 (0)