Skip to content

Commit deaf3be

Browse files
srishtipmishrasrishtipmishra
committed
Add support for application definition parameters update in dataflow application
1 parent 0103e98 commit deaf3be

File tree

4 files changed

+57
-27
lines changed

4 files changed

+57
-27
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
### Added
44
- Support for pod security policy in kubernetes
55
- Support for Oracle Big Data Service
6+
- Support for application definition parameters update in dataflow application
67

78
## 3.68.0 (March 25, 2020)
89

‎oci/dataflow_application_resource.go‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,12 +47,10 @@ func DataflowApplicationResource() *schema.Resource {
4747
"file_uri": {
4848
Type: schema.TypeString,
4949
Required: true,
50-
ForceNew: true,
5150
},
5251
"language": {
5352
Type: schema.TypeString,
5453
Required: true,
55-
ForceNew: true,
5654
},
5755
"num_executors": {
5856
Type: schema.TypeInt,
@@ -61,7 +59,6 @@ func DataflowApplicationResource() *schema.Resource {
6159
"spark_version": {
6260
Type: schema.TypeString,
6361
Required: true,
64-
ForceNew: true,
6562
},
6663

6764
// Optional
@@ -76,8 +73,6 @@ func DataflowApplicationResource() *schema.Resource {
7673
"class_name": {
7774
Type: schema.TypeString,
7875
Optional: true,
79-
Computed: true,
80-
ForceNew: true,
8176
},
8277
"configuration": {
8378
Type: schema.TypeMap,
@@ -380,6 +375,11 @@ func (s *DataflowApplicationResourceCrud) Update() error {
380375
}
381376
}
382377

378+
if className, ok := s.D.GetOkExists("class_name"); ok {
379+
tmp := className.(string)
380+
request.ClassName = &tmp
381+
}
382+
383383
if configuration, ok := s.D.GetOkExists("configuration"); ok {
384384
request.Configuration = objectMapToStringMap(configuration.(map[string]interface{}))
385385
}
@@ -412,10 +412,19 @@ func (s *DataflowApplicationResourceCrud) Update() error {
412412
request.ExecutorShape = &tmp
413413
}
414414

415+
if fileUri, ok := s.D.GetOkExists("file_uri"); ok {
416+
tmp := fileUri.(string)
417+
request.FileUri = &tmp
418+
}
419+
415420
if freeformTags, ok := s.D.GetOkExists("freeform_tags"); ok {
416421
request.FreeformTags = objectMapToStringMap(freeformTags.(map[string]interface{}))
417422
}
418423

424+
if language, ok := s.D.GetOkExists("language"); ok {
425+
request.Language = oci_dataflow.ApplicationLanguageEnum(language.(string))
426+
}
427+
419428
if logsBucketUri, ok := s.D.GetOkExists("logs_bucket_uri"); ok {
420429
tmp := logsBucketUri.(string)
421430
request.LogsBucketUri = &tmp
@@ -443,6 +452,11 @@ func (s *DataflowApplicationResourceCrud) Update() error {
443452
}
444453
}
445454

455+
if sparkVersion, ok := s.D.GetOkExists("spark_version"); ok {
456+
tmp := sparkVersion.(string)
457+
request.SparkVersion = &tmp
458+
}
459+
446460
if warehouseBucketUri, ok := s.D.GetOkExists("warehouse_bucket_uri"); ok {
447461
tmp := warehouseBucketUri.(string)
448462
request.WarehouseBucketUri = &tmp

‎oci/dataflow_application_test.go‎

Lines changed: 33 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,10 @@ var (
4343
"display_name": Representation{repType: Required, create: `test_wordcount_app`, update: `displayName2`},
4444
"driver_shape": Representation{repType: Required, create: `VM.Standard2.1`},
4545
"executor_shape": Representation{repType: Required, create: `VM.Standard2.1`},
46-
"file_uri": Representation{repType: Required, create: `${var.dataflow_file_uri}`},
47-
"language": Representation{repType: Required, create: `PYTHON`},
46+
"file_uri": Representation{repType: Required, create: `${var.dataflow_file_uri}`, update: `${var.dataflow_file_uri_updated}`},
47+
"language": Representation{repType: Required, create: `PYTHON`, update: `SCALA`},
4848
"num_executors": Representation{repType: Required, create: `1`, update: `2`},
49-
"spark_version": Representation{repType: Required, create: `2.4`},
49+
"spark_version": Representation{repType: Required, create: `2.4`, update: `2.4.4`},
5050
"arguments": Representation{repType: Optional, create: []string{`arguments`}, update: []string{`arguments2`}},
5151
"configuration": Representation{repType: Optional, create: map[string]string{"spark.shuffle.io.maxRetries": "10"}, update: map[string]string{"spark.shuffle.io.maxRetries": "11"}},
5252
"defined_tags": Representation{repType: Optional, create: `${map("${oci_identity_tag_namespace.tag-namespace1.name}.${oci_identity_tag.tag1.name}", "value")}`, update: `${map("${oci_identity_tag_namespace.tag-namespace1.name}.${oci_identity_tag.tag1.name}", "updatedValue")}`},
@@ -75,12 +75,16 @@ func TestDataflowApplicationResource_basic(t *testing.T) {
7575
compartmentIdVariableStr := fmt.Sprintf("variable \"compartment_id\" { default = \"%s\" }\n", compartmentId)
7676

7777
fileUri := getEnvSettingWithBlankDefault("dataflow_file_uri")
78+
fileUriUpdated := getEnvSettingWithBlankDefault("dataflow_file_uri_updated")
7879
fileUriVariableStr := fmt.Sprintf("variable \"dataflow_file_uri\" { default = \"%s\" }\n", fileUri)
80+
fileUriVariableStrUpdated := fmt.Sprintf("variable \"dataflow_file_uri_updated\" { default = \"%s\" }\n", fileUriUpdated)
81+
7982
logsBucketUri := getEnvSettingWithBlankDefault("dataflow_logs_bucket_uri")
8083
logsBucketUriVariableStr := fmt.Sprintf("variable \"dataflow_logs_bucket_uri\" { default = \"%s\" }\n", logsBucketUri)
8184
warehouseBucketUri := getEnvSettingWithBlankDefault("dataflow_warehouse_bucket_uri")
8285
warehouseBucketUriVariableStr := fmt.Sprintf("variable \"dataflow_warehouse_bucket_uri\" { default = \"%s\" }\n", warehouseBucketUri)
83-
86+
classNameUpdated := getEnvSettingWithBlankDefault("dataflow_class_name_updated")
87+
classNameStrUpdated := fmt.Sprintf("variable \"dataflow_class_name_updated\" { default = \"%s\" }\n", classNameUpdated)
8488
resourceName := "oci_dataflow_application.test_application"
8589
datasourceName := "data.oci_dataflow_applications.test_applications"
8690
singularDatasourceName := "data.oci_dataflow_application.test_application"
@@ -162,8 +166,11 @@ func TestDataflowApplicationResource_basic(t *testing.T) {
162166

163167
// verify updates to updatable parameters
164168
{
165-
Config: config + compartmentIdVariableStr + fileUriVariableStr + logsBucketUriVariableStr + warehouseBucketUriVariableStr + dataFlowApplicationResourceDependencies +
166-
generateResourceFromRepresentationMap("oci_dataflow_application", "test_application", Optional, Update, dataFlowApplicationRepresentation),
169+
Config: config + compartmentIdVariableStr + fileUriVariableStr + classNameStrUpdated + fileUriVariableStrUpdated + logsBucketUriVariableStr + warehouseBucketUriVariableStr + dataFlowApplicationResourceDependencies +
170+
generateResourceFromRepresentationMap("oci_dataflow_application", "test_application", Optional, Update,
171+
representationCopyWithNewProperties(dataFlowApplicationRepresentation, map[string]interface{}{
172+
"class_name": Representation{repType: Optional, create: `${var.dataflow_class_name_updated}`},
173+
})),
167174
Check: resource.ComposeAggregateTestCheckFunc(
168175
resource.TestCheckResourceAttr(resourceName, "arguments.#", "1"),
169176
resource.TestCheckResourceAttr(resourceName, "compartment_id", compartmentId),
@@ -173,21 +180,22 @@ func TestDataflowApplicationResource_basic(t *testing.T) {
173180
resource.TestCheckResourceAttr(resourceName, "display_name", "displayName2"),
174181
resource.TestCheckResourceAttr(resourceName, "driver_shape", "VM.Standard2.1"),
175182
resource.TestCheckResourceAttr(resourceName, "executor_shape", "VM.Standard2.1"),
176-
resource.TestCheckResourceAttr(resourceName, "file_uri", fileUri),
183+
resource.TestCheckResourceAttr(resourceName, "file_uri", fileUriUpdated),
177184
resource.TestCheckResourceAttr(resourceName, "freeform_tags.%", "1"),
178185
resource.TestCheckResourceAttrSet(resourceName, "id"),
179-
resource.TestCheckResourceAttr(resourceName, "language", "PYTHON"),
186+
resource.TestCheckResourceAttr(resourceName, "language", "SCALA"),
180187
resource.TestCheckResourceAttr(resourceName, "logs_bucket_uri", logsBucketUri),
181188
resource.TestCheckResourceAttr(resourceName, "num_executors", "2"),
182189
resource.TestCheckResourceAttrSet(resourceName, "owner_principal_id"),
183190
resource.TestCheckResourceAttr(resourceName, "parameters.#", "1"),
184191
resource.TestCheckResourceAttr(resourceName, "parameters.0.name", "name2"),
185192
resource.TestCheckResourceAttr(resourceName, "parameters.0.value", "value2"),
186-
resource.TestCheckResourceAttr(resourceName, "spark_version", "2.4"),
193+
resource.TestCheckResourceAttr(resourceName, "spark_version", "2.4.4"),
187194
resource.TestCheckResourceAttrSet(resourceName, "state"),
188195
resource.TestCheckResourceAttrSet(resourceName, "time_created"),
189196
resource.TestCheckResourceAttrSet(resourceName, "time_updated"),
190197
resource.TestCheckResourceAttr(resourceName, "warehouse_bucket_uri", warehouseBucketUri),
198+
resource.TestCheckResourceAttr(resourceName, "class_name", classNameUpdated),
191199

192200
func(s *terraform.State) (err error) {
193201
resId2, err = fromInstanceState(s, resourceName, "id")
@@ -202,19 +210,20 @@ func TestDataflowApplicationResource_basic(t *testing.T) {
202210
{
203211
Config: config +
204212
generateDataSourceFromRepresentationMap("oci_dataflow_applications", "test_applications", Optional, Update, dataFlowApplicationDataSourceRepresentation) +
205-
compartmentIdVariableStr + fileUriVariableStr + logsBucketUriVariableStr + warehouseBucketUriVariableStr + dataFlowApplicationResourceDependencies +
206-
generateResourceFromRepresentationMap("oci_dataflow_application", "test_application", Optional, Update, dataFlowApplicationRepresentation),
213+
compartmentIdVariableStr + fileUriVariableStr + fileUriVariableStrUpdated + logsBucketUriVariableStr + classNameStrUpdated + warehouseBucketUriVariableStr + dataFlowApplicationResourceDependencies +
214+
generateResourceFromRepresentationMap("oci_dataflow_application", "test_application", Optional, Update, representationCopyWithNewProperties(dataFlowApplicationRepresentation, map[string]interface{}{
215+
"class_name": Representation{repType: Optional, create: `${var.dataflow_class_name_updated}`},
216+
})),
207217
Check: resource.ComposeAggregateTestCheckFunc(
208218
resource.TestCheckResourceAttr(datasourceName, "compartment_id", compartmentId),
209219
resource.TestCheckResourceAttr(datasourceName, "display_name", "displayName2"),
210-
211220
resource.TestCheckResourceAttr(datasourceName, "applications.#", "1"),
212221
resource.TestCheckResourceAttr(datasourceName, "applications.0.compartment_id", compartmentId),
213222
resource.TestCheckResourceAttr(datasourceName, "applications.0.defined_tags.%", "1"),
214223
resource.TestCheckResourceAttr(datasourceName, "applications.0.display_name", "displayName2"),
215224
resource.TestCheckResourceAttr(datasourceName, "applications.0.freeform_tags.%", "1"),
216225
resource.TestCheckResourceAttrSet(datasourceName, "applications.0.id"),
217-
resource.TestCheckResourceAttr(datasourceName, "applications.0.language", "PYTHON"),
226+
resource.TestCheckResourceAttr(datasourceName, "applications.0.language", "SCALA"),
218227
resource.TestCheckResourceAttrSet(datasourceName, "applications.0.owner_principal_id"),
219228
resource.TestCheckResourceAttrSet(datasourceName, "applications.0.owner_user_name"),
220229
resource.TestCheckResourceAttrSet(datasourceName, "applications.0.state"),
@@ -226,7 +235,10 @@ func TestDataflowApplicationResource_basic(t *testing.T) {
226235
{
227236
Config: config +
228237
generateDataSourceFromRepresentationMap("oci_dataflow_application", "test_application", Required, Create, dataFlowApplicationSingularDataSourceRepresentation) +
229-
compartmentIdVariableStr + fileUriVariableStr + logsBucketUriVariableStr + warehouseBucketUriVariableStr + DataFlowApplicationResourceConfig,
238+
compartmentIdVariableStr + fileUriVariableStr + fileUriVariableStrUpdated + logsBucketUriVariableStr + classNameStrUpdated + warehouseBucketUriVariableStr + dataFlowApplicationResourceDependencies +
239+
generateResourceFromRepresentationMap("oci_dataflow_application", "test_application", Optional, Update, representationCopyWithNewProperties(dataFlowApplicationRepresentation, map[string]interface{}{
240+
"class_name": Representation{repType: Optional, create: `${var.dataflow_class_name_updated}`},
241+
})),
230242
Check: resource.ComposeAggregateTestCheckFunc(
231243
resource.TestCheckResourceAttrSet(singularDatasourceName, "application_id"),
232244

@@ -238,17 +250,17 @@ func TestDataflowApplicationResource_basic(t *testing.T) {
238250
resource.TestCheckResourceAttr(singularDatasourceName, "display_name", "displayName2"),
239251
resource.TestCheckResourceAttr(singularDatasourceName, "driver_shape", "VM.Standard2.1"),
240252
resource.TestCheckResourceAttr(singularDatasourceName, "executor_shape", "VM.Standard2.1"),
241-
resource.TestCheckResourceAttr(singularDatasourceName, "file_uri", fileUri),
253+
resource.TestCheckResourceAttr(singularDatasourceName, "file_uri", fileUriUpdated),
242254
resource.TestCheckResourceAttr(singularDatasourceName, "freeform_tags.%", "1"),
243255
resource.TestCheckResourceAttrSet(singularDatasourceName, "id"),
244-
resource.TestCheckResourceAttr(singularDatasourceName, "language", "PYTHON"),
256+
resource.TestCheckResourceAttr(singularDatasourceName, "language", "SCALA"),
245257
resource.TestCheckResourceAttr(singularDatasourceName, "logs_bucket_uri", logsBucketUri),
246258
resource.TestCheckResourceAttr(singularDatasourceName, "num_executors", "2"),
247259
resource.TestCheckResourceAttrSet(singularDatasourceName, "owner_user_name"),
248260
resource.TestCheckResourceAttr(singularDatasourceName, "parameters.#", "1"),
249261
resource.TestCheckResourceAttr(singularDatasourceName, "parameters.0.name", "name2"),
250262
resource.TestCheckResourceAttr(singularDatasourceName, "parameters.0.value", "value2"),
251-
resource.TestCheckResourceAttr(singularDatasourceName, "spark_version", "2.4"),
263+
resource.TestCheckResourceAttr(singularDatasourceName, "spark_version", "2.4.4"),
252264
resource.TestCheckResourceAttrSet(singularDatasourceName, "state"),
253265
resource.TestCheckResourceAttrSet(singularDatasourceName, "time_created"),
254266
resource.TestCheckResourceAttrSet(singularDatasourceName, "time_updated"),
@@ -257,7 +269,10 @@ func TestDataflowApplicationResource_basic(t *testing.T) {
257269
},
258270
// remove singular datasource from previous step so that it doesn't conflict with import tests
259271
{
260-
Config: config + compartmentIdVariableStr + fileUriVariableStr + logsBucketUriVariableStr + warehouseBucketUriVariableStr + DataFlowApplicationResourceConfig,
272+
Config: config + compartmentIdVariableStr + fileUriVariableStr + fileUriVariableStrUpdated + classNameStrUpdated + logsBucketUriVariableStr + warehouseBucketUriVariableStr + dataFlowApplicationResourceDependencies +
273+
generateResourceFromRepresentationMap("oci_dataflow_application", "test_application", Optional, Update, representationCopyWithNewProperties(dataFlowApplicationRepresentation, map[string]interface{}{
274+
"class_name": Representation{repType: Optional, create: `${var.dataflow_class_name_updated}`},
275+
})),
261276
},
262277
// verify resource import
263278
{

‎website/docs/r/dataflow_application.html.markdown‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,23 +49,23 @@ resource "oci_dataflow_application" "test_application" {
4949
The following arguments are supported:
5050

5151
* `arguments` - (Optional) (Updatable) The arguments passed to the running application as command line arguments. An argument is either a plain text or a placeholder. Placeholders are replaced using values from the parameters map. Each placeholder specified must be represented in the parameters map else the request (POST or PUT) will fail with a HTTP 400 status code. Placeholders are specified as `Service Api Spec`, where `name` is the name of the parameter. Example: `[ "--input", "${input_file}", "--name", "John Doe" ]` If "input_file" has a value of "mydata.xml", then the value above will be translated to `--input mydata.xml --name "John Doe"`
52-
* `class_name` - (Optional) The class for the application.
52+
* `class_name` - (Optional) (Updatable) The class for the application.
5353
* `compartment_id` - (Required) The OCID of the compartment that contains this application.
5454
* `configuration` - (Optional) (Updatable) The Spark configuration passed to the running process. See https://spark.apache.org/docs/latest/configuration.html#available-properties Example: { "spark.app.name" : "My App Name", "spark.shuffle.io.maxRetries" : "4" } Note: Not all Spark properties are permitted to be set. Attempting to set a property that is not allowed to be overwritten will cause a 400 status to be returned.
5555
* `defined_tags` - (Optional) (Updatable) Defined tags for this resource. Each key is predefined and scoped to a namespace. For more information, see [Resource Tags](https://docs.cloud.oracle.com/iaas/Content/General/Concepts/resourcetags.htm). Example: `{"Operations.CostCenter": "42"}`
5656
* `description` - (Optional) (Updatable) A user-friendly description. Avoid entering confidential information.
5757
* `display_name` - (Required) (Updatable) A user-friendly name. It does not have to be unique. Avoid entering confidential information.
5858
* `driver_shape` - (Required) (Updatable) The VM shape for the driver. Sets the driver cores and memory.
5959
* `executor_shape` - (Required) (Updatable) The VM shape for the executors. Sets the executor cores and memory.
60-
* `file_uri` - (Required) An Oracle Cloud Infrastructure URI of the file containing the application to execute. See https://docs.cloud.oracle.com/iaas/Content/API/SDKDocs/hdfsconnector.htm#uriformat
60+
* `file_uri` - (Required) (Updatable) An Oracle Cloud Infrastructure URI of the file containing the application to execute. See https://docs.cloud.oracle.com/iaas/Content/API/SDKDocs/hdfsconnector.htm#uriformat
6161
* `freeform_tags` - (Optional) (Updatable) Free-form tags for this resource. Each tag is a simple key-value pair with no predefined name, type, or namespace. For more information, see [Resource Tags](https://docs.cloud.oracle.com/iaas/Content/General/Concepts/resourcetags.htm). Example: `{"Department": "Finance"}`
62-
* `language` - (Required) The Spark language.
62+
* `language` - (Required) (Updatable) The Spark language.
6363
* `logs_bucket_uri` - (Optional) (Updatable) An Oracle Cloud Infrastructure URI of the bucket where the Spark job logs are to be uploaded. See https://docs.cloud.oracle.com/iaas/Content/API/SDKDocs/hdfsconnector.htm#uriformat
6464
* `num_executors` - (Required) (Updatable) The number of executor VMs requested.
6565
* `parameters` - (Optional) (Updatable) An array of name/value pairs used to fill placeholders found in properties like `Application.arguments`. The name must be a string of one or more word characters (a-z, A-Z, 0-9, _). The value can be a string of 0 or more characters of any kind. Example: [ { name: "iterations", value: "10"}, { name: "input_file", value: "mydata.xml" }, { name: "variable_x", value: "${x}"} ]
6666
* `name` - (Required) (Updatable) The name of the parameter. It must be a string of one or more word characters (a-z, A-Z, 0-9, _). Examples: "iterations", "input_file"
6767
* `value` - (Required) (Updatable) The value of the parameter. It must be a string of 0 or more characters of any kind. Examples: "" (empty string), "10", "mydata.xml", "${x}"
68-
* `spark_version` - (Required) The Spark version utilized to run the application.
68+
* `spark_version` - (Required) (Updatable) The Spark version utilized to run the application.
6969
* `warehouse_bucket_uri` - (Optional) (Updatable) An Oracle Cloud Infrastructure URI of the bucket to be used as default warehouse directory for BATCH SQL runs. See https://docs.cloud.oracle.com/iaas/Content/API/SDKDocs/hdfsconnector.htm#uriformat
7070

7171

0 commit comments

Comments
 (0)