Add base64_encode_content attribute to datasources that may return binary content

alexng-canuck · alexng-canuck · commit 9d23bf4c8d38 · 2019-07-08T17:04:45.000-07:00
This field should workaround v0.12 regression that causes binary content to be corrupted when written directly to state.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,7 @@
 - Support for Granular Security Lists using Network Security Group
 - Support for Granular Security Lists in Load Balancer
 - Support for Network Security Groups in databases
+- Support in autonomous database and object data sources for encoding downloaded binary content as base64. This works around behavior in Terraform v0.12 that could cause binary content to be corrupted if written directly to state.
 
 ### Fixed
 - Address panics caused by invalid type assertions in object map conversion. This could potentially affect attributes
diff --git a/examples/database/adb/autonomous_data_warehouse_wallet.tf b/examples/database/adb/autonomous_data_warehouse_wallet.tf
@@ -6,14 +6,14 @@ resource "random_string" "autonomous_data_warehouse_wallet_password" {
 }
 
 data "oci_database_autonomous_database_wallet" "autonomous_data_warehouse_wallet" {
-  #Required
   autonomous_database_id = "${oci_database_autonomous_database.autonomous_data_warehouse.id}"
   password               = "${random_string.autonomous_data_warehouse_wallet_password.result}"
+  base64_encode_content  = "true"
 }
 
 resource "local_file" "autonomous_data_warehouse_wallet_file" {
-  content  = "${data.oci_database_autonomous_database_wallet.autonomous_data_warehouse_wallet.content}"
-  filename = "${path.module}/autonomous_data_warehouse_wallet.zip"
+  content_base64 = "${data.oci_database_autonomous_database_wallet.autonomous_data_warehouse_wallet.content}"
+  filename       = "${path.module}/autonomous_data_warehouse_wallet.zip"
 }
 
 output "autonomous_data_warehouse_wallet_password" {
diff --git a/examples/database/adb/autonomous_database_wallet.tf b/examples/database/adb/autonomous_database_wallet.tf
@@ -6,14 +6,14 @@ resource "random_string" "autonomous_database_wallet_password" {
 }
 
 data "oci_database_autonomous_database_wallet" "autonomous_database_wallet" {
-  #Required
   autonomous_database_id = "${oci_database_autonomous_database.autonomous_database.id}"
   password               = "${random_string.autonomous_database_wallet_password.result}"
+  base64_encode_content  = "true"
 }
 
 resource "local_file" "autonomous_database_wallet_file" {
-  content  = "${data.oci_database_autonomous_database_wallet.autonomous_database_wallet.content}"
-  filename = "${path.module}/autonomous_database_wallet.zip"
+  content_base64 = "${data.oci_database_autonomous_database_wallet.autonomous_database_wallet.content}"
+  filename       = "${path.module}/autonomous_database_wallet.zip"
 }
 
 output "autonomous_database_wallet_password" {
diff --git a/examples/database/adb/provider.tf b/examples/database/adb/provider.tf
@@ -7,3 +7,7 @@ provider "oci" {
   private_key_path = "${var.private_key_path}"
   region           = "${var.region}"
 }
+
+provider "local" {
+  version = ">=1.3.0" # Need this version of the local provider to support base64 encoded inputs
+}
diff --git a/oci/database_autonomous_data_warehouse_wallet_data_source.go b/oci/database_autonomous_data_warehouse_wallet_data_source.go
@@ -4,6 +4,7 @@ package provider
 
 import (
 	"context"
+	"encoding/base64"
 	"io/ioutil"
 
 	"github.com/hashicorp/terraform/helper/schema"
@@ -24,6 +25,11 @@ func DatabaseAutonomousDataWarehouseWalletDataSource() *schema.Resource {
 				Required:  true,
 				Sensitive: true,
 			},
+			"base64_encode_content": {
+				Type:     schema.TypeBool,
+				Optional: true,
+				Default:  false,
+			},
 
 			// Computed
 			"content": {
@@ -91,7 +97,18 @@ func (s *DatabaseAutonomousDataWarehouseWalletDataSourceCrud) SetData() error {
 
 	s.D.SetId(GenerateDataSourceID())
 
-	s.D.Set("content", string(*s.Res))
+	base64EncodeContent := false
+	if tmp, ok := s.D.GetOkExists("base64_encode_content"); ok {
+		base64EncodeContent = tmp.(bool)
+	}
+
+	if base64EncodeContent {
+		// This use case is for v0.12, where content should be base64 encoded to avoid
+		// being normalized before setting in state. Otherwise, the zip file may get corrupted.
+		s.D.Set("content", base64.StdEncoding.EncodeToString(*s.Res))
+	} else {
+		s.D.Set("content", string(*s.Res))
+	}
 
 	return nil
 }
diff --git a/oci/database_autonomous_data_warehouse_wallet_test.go b/oci/database_autonomous_data_warehouse_wallet_test.go
@@ -16,6 +16,7 @@ var (
 	autonomousDataWarehouseWalletSingularDataSourceRepresentation = map[string]interface{}{
 		"autonomous_data_warehouse_id": Representation{repType: Required, create: `${oci_database_autonomous_data_warehouse.test_autonomous_data_warehouse.id}`},
 		"password":                     Representation{repType: Required, create: `BEstrO0ng_#11`},
+		"base64_encode_content":        Representation{repType: Optional, create: `true`},
 	}
 
 	AutonomousDataWarehouseWalletResourceConfig = AutonomousDataWarehouseResourceConfig
@@ -47,9 +48,22 @@ func TestDatabaseAutonomousDataWarehouseWalletResource_basic(t *testing.T) {
 				Check: resource.ComposeAggregateTestCheckFunc(
 					resource.TestCheckResourceAttrSet(singularDatasourceName, "autonomous_data_warehouse_id"),
 					resource.TestCheckResourceAttr(singularDatasourceName, "password", "BEstrO0ng_#11"),
+					resource.TestCheckResourceAttr(singularDatasourceName, "base64_encode_content", "false"),
 					resource.TestCheckResourceAttrSet(singularDatasourceName, "content"),
 				),
 			},
+			// verify datasource with base64 encoded content
+			{
+				Config: config +
+					generateDataSourceFromRepresentationMap("oci_database_autonomous_data_warehouse_wallet", "test_autonomous_data_warehouse_wallet", Optional, Create, autonomousDataWarehouseWalletSingularDataSourceRepresentation) +
+					compartmentIdVariableStr + AutonomousDataWarehouseWalletResourceConfig,
+				Check: resource.ComposeAggregateTestCheckFunc(
+					resource.TestCheckResourceAttrSet(singularDatasourceName, "autonomous_data_warehouse_id"),
+					resource.TestCheckResourceAttr(singularDatasourceName, "password", "BEstrO0ng_#11"),
+					resource.TestCheckResourceAttr(singularDatasourceName, "base64_encode_content", "true"),
+					testCheckAttributeBase64Encoded(singularDatasourceName, "content", true),
+				),
+			},
 		},
 	})
 }
diff --git a/oci/database_autonomous_database_wallet_data_source.go b/oci/database_autonomous_database_wallet_data_source.go
@@ -4,6 +4,7 @@ package provider
 
 import (
 	"context"
+	"encoding/base64"
 	"io/ioutil"
 
 	"github.com/hashicorp/terraform/helper/schema"
@@ -24,6 +25,11 @@ func DatabaseAutonomousDatabaseWalletDataSource() *schema.Resource {
 				Required:  true,
 				Sensitive: true,
 			},
+			"base64_encode_content": {
+				Type:     schema.TypeBool,
+				Optional: true,
+				Default:  false,
+			},
 
 			// Computed
 			"content": {
@@ -91,7 +97,18 @@ func (s *DatabaseAutonomousDatabaseWalletDataSourceCrud) SetData() error {
 
 	s.D.SetId(GenerateDataSourceID())
 
-	s.D.Set("content", string(*s.Res))
+	base64EncodeContent := false
+	if tmp, ok := s.D.GetOkExists("base64_encode_content"); ok {
+		base64EncodeContent = tmp.(bool)
+	}
+
+	if base64EncodeContent {
+		// This use case is for v0.12, where content should be base64 encoded to avoid
+		// being normalized before setting in state. Otherwise, the zip file may get corrupted.
+		s.D.Set("content", base64.StdEncoding.EncodeToString(*s.Res))
+	} else {
+		s.D.Set("content", string(*s.Res))
+	}
 
 	return nil
 }
diff --git a/oci/database_autonomous_database_wallet_test.go b/oci/database_autonomous_database_wallet_test.go
@@ -16,6 +16,7 @@ var (
 	autonomousDatabaseWalletSingularDataSourceRepresentation = map[string]interface{}{
 		"autonomous_database_id": Representation{repType: Required, create: `${oci_database_autonomous_database.test_autonomous_database.id}`},
 		"password":               Representation{repType: Required, create: `BEstrO0ng_#11`},
+		"base64_encode_content":  Representation{repType: Optional, create: `true`},
 	}
 
 	AutonomousDatabaseWalletResourceConfig = AutonomousDatabaseResourceConfig
@@ -47,9 +48,22 @@ func TestDatabaseAutonomousDatabaseWalletResource_basic(t *testing.T) {
 				Check: resource.ComposeAggregateTestCheckFunc(
 					resource.TestCheckResourceAttrSet(singularDatasourceName, "autonomous_database_id"),
 					resource.TestCheckResourceAttr(singularDatasourceName, "password", "BEstrO0ng_#11"),
+					resource.TestCheckResourceAttr(singularDatasourceName, "base64_encode_content", "false"),
 					resource.TestCheckResourceAttrSet(singularDatasourceName, "content"),
 				),
 			},
+			// verify datasource with base64 encoded content
+			{
+				Config: config +
+					generateDataSourceFromRepresentationMap("oci_database_autonomous_database_wallet", "test_autonomous_database_wallet", Optional, Create, autonomousDatabaseWalletSingularDataSourceRepresentation) +
+					compartmentIdVariableStr + AutonomousDatabaseWalletResourceConfig,
+				Check: resource.ComposeAggregateTestCheckFunc(
+					resource.TestCheckResourceAttrSet(singularDatasourceName, "autonomous_database_id"),
+					resource.TestCheckResourceAttr(singularDatasourceName, "password", "BEstrO0ng_#11"),
+					resource.TestCheckResourceAttr(singularDatasourceName, "base64_encode_content", "true"),
+					testCheckAttributeBase64Encoded(singularDatasourceName, "content", true),
+				),
+			},
 		},
 	})
 }
diff --git a/oci/object_storage_object_data_source.go b/oci/object_storage_object_data_source.go
@@ -4,6 +4,7 @@ package provider
 
 import (
 	"context"
+	"encoding/base64"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -35,6 +36,11 @@ func ObjectStorageObjectDataSource() *schema.Resource {
 				//default value is 1MB
 				Default: 1048576,
 			},
+			"base64_encode_content": {
+				Type:     schema.TypeBool,
+				Optional: true,
+				Default:  false,
+			},
 
 			// Computed
 			"content": {
@@ -144,10 +150,19 @@ func (s *ObjectStorageObjectDataSourceCrud) SetData() error {
 
 	s.D.SetId(GenerateDataSourceID())
 
+	base64EncodeContent := false
+	if tmp, ok := s.D.GetOkExists("base64_encode_content"); ok {
+		base64EncodeContent = tmp.(bool)
+	}
+
 	contentReader := s.Res.Content
 	contentArray, err := ioutil.ReadAll(contentReader)
 	if err != nil {
 		log.Printf("unable to read 'content' from response. Error: %v", err)
+	} else if base64EncodeContent {
+		// This use case is for v0.12, where content should be base64 encoded to avoid
+		// being normalized before setting in state.
+		s.D.Set("content", base64.StdEncoding.EncodeToString(contentArray))
 	} else {
 		s.D.Set("content", string(contentArray))
 	}
diff --git a/oci/object_storage_object_test.go b/oci/object_storage_object_test.go
@@ -5,6 +5,7 @@ package provider
 import (
 	"context"
 	"crypto/md5"
+	"encoding/base64"
 	"encoding/hex"
 	"fmt"
 	"io/ioutil"
@@ -42,10 +43,11 @@ var (
 	}
 
 	objectSingularDataSourceRepresentation = map[string]interface{}{
-		"bucket":               Representation{repType: Required, create: `${oci_objectstorage_bucket.test_bucket.name}`},
-		"namespace":            Representation{repType: Required, create: `${oci_objectstorage_bucket.test_bucket.namespace}`},
-		"object":               Representation{repType: Required, create: `my-test-object-3`},
-		"content_length_limit": Representation{repType: Optional, create: `17`, update: `15`},
+		"bucket":                Representation{repType: Required, create: `${oci_objectstorage_bucket.test_bucket.name}`},
+		"namespace":             Representation{repType: Required, create: `${oci_objectstorage_bucket.test_bucket.namespace}`},
+		"object":                Representation{repType: Required, create: `my-test-object-3`},
+		"content_length_limit":  Representation{repType: Optional, create: `17`, update: `15`},
+		"base64_encode_content": Representation{repType: Optional, create: `true`},
 	}
 
 	objectRepresentation = map[string]interface{}{
@@ -204,6 +206,7 @@ func TestObjectStorageObjectResource_basic(t *testing.T) {
 						getUpdatedRepresentationCopy("object", Representation{repType: Required, create: `my-test-object-1`, update: `my-test-object-3`}, objectRepresentation)) +
 					generateDataSourceFromRepresentationMap("oci_objectstorage_object", "test_object", Required, Create, objectSingularDataSourceRepresentation),
 				Check: resource.ComposeAggregateTestCheckFunc(
+					resource.TestCheckResourceAttr(singularDatasourceName, "base64_encode_content", "false"),
 					resource.TestCheckResourceAttr(singularDatasourceName, "content_encoding", "identity"),
 					resource.TestCheckResourceAttr(singularDatasourceName, "content_language", "en-CA"),
 					resource.TestCheckResourceAttr(singularDatasourceName, "content_length", "16"),
@@ -218,20 +221,22 @@ func TestObjectStorageObjectResource_basic(t *testing.T) {
 					resource.TestCheckResourceAttr(singularDatasourceName, "object", "my-test-object-3"),
 				),
 			},
+			// verify base64 encoding in singular datasource
 			{
 				Config: config + compartmentIdVariableStr + ObjectResourceDependencies +
 					generateResourceFromRepresentationMap("oci_objectstorage_object", "test_object", Optional, Update,
 						getUpdatedRepresentationCopy("object", Representation{repType: Required, create: `my-test-object-1`, update: `my-test-object-3`}, objectRepresentation)) +
 					generateDataSourceFromRepresentationMap("oci_objectstorage_object", "test_object", Optional, Create, objectSingularDataSourceRepresentation),
 				Check: resource.ComposeAggregateTestCheckFunc(
+					resource.TestCheckResourceAttr(singularDatasourceName, "base64_encode_content", "true"),
 					resource.TestCheckResourceAttr(singularDatasourceName, "content_encoding", "identity"),
 					resource.TestCheckResourceAttr(singularDatasourceName, "content_language", "en-CA"),
 					resource.TestCheckResourceAttr(singularDatasourceName, "content_length", "16"),
 					resource.TestCheckResourceAttrSet(singularDatasourceName, "content_md5"),
 					resource.TestCheckResourceAttr(singularDatasourceName, "content_type", "text/xml"),
 					resource.TestCheckResourceAttr(singularDatasourceName, "bucket", testBucketName),
 					resource.TestCheckResourceAttrSet(singularDatasourceName, "content"),
-					resource.TestCheckResourceAttr(singularDatasourceName, "content", "<a1>content</a1>"),
+					resource.TestCheckResourceAttr(singularDatasourceName, "content", base64.StdEncoding.EncodeToString([]byte("<a1>content</a1>"))),
 					resource.TestCheckResourceAttr(singularDatasourceName, "metadata.%", "1"),
 					resource.TestCheckResourceAttr(singularDatasourceName, "metadata.content-type", "text/xml"),
 					resource.TestCheckResourceAttrSet(singularDatasourceName, "namespace"),
diff --git a/oci/test_helpers.go b/oci/test_helpers.go
@@ -4,6 +4,7 @@ package provider
 
 import (
 	"bytes"
+	"encoding/base64"
 	"fmt"
 	"log"
 	"os"
@@ -104,6 +105,29 @@ func TestCheckResourceAttributesEqual(name1, key1, name2, key2 string) resource.
 	}
 }
 
+func testCheckAttributeBase64Encoded(name, key string, expectBase64Encoded bool) resource.TestCheckFunc {
+	return func(s *terraform.State) error {
+		content, err := fromInstanceState(s, name, key)
+		if err != nil {
+			return err
+		}
+
+		isBase64Encoded := true
+		if _, err := base64.StdEncoding.DecodeString(content); err != nil {
+			isBase64Encoded = false
+		}
+
+		if isBase64Encoded != expectBase64Encoded {
+			if expectBase64Encoded {
+				return fmt.Errorf("expected '%s' to be base64 encoded, but it is not", key)
+			}
+			return fmt.Errorf("expected '%s' to not be base64 encoded, but it is", key)
+		}
+
+		return nil
+	}
+}
+
 // Function to be implemented by resources that wish to wait on a certain condition and this function is responsible for evaluating the specific condition for that resource
 type ShouldWaitFunc func(response oci_common.OCIOperationResponse) bool
 
diff --git a/website/docs/d/database_autonomous_data_warehouse_wallet.html.markdown b/website/docs/d/database_autonomous_data_warehouse_wallet.html.markdown
@@ -28,10 +28,12 @@ The following arguments are supported:
 
 * `autonomous_data_warehouse_id` - (Required) The database [OCID](https://docs.cloud.oracle.com/iaas/Content/General/Concepts/identifiers.htm).
 * `password` - (Required) The password to encrypt the keys inside the wallet. The password must be at least 8 characters long and must include at least 1 letter and either 1 numeric character or 1 special character.
-
+* `base64_encode_content` - (Optional) Encodes the downloaded zipped wallet in base64. It is recommended to set this to `true` to avoid corrupting the zip file in Terraform state. The default value is `false` to preserve backwards compatibility with Terraform v0.11 configurations.
 
 ## Attributes Reference
 
 The following attributes are exported:
 
-* `content` - content of the downloaded zipped wallet for the Autonomous Data Warehouse
+* `content` - content of the downloaded zipped wallet for the Autonomous Data Warehouse. If `base64_encode_content` is set to `true`, then this content will be base64 encoded.
+If passing the base64 encoded content to a `local_file` resource, please use the `content_base64` attribute of the `local_file` resource.
+See this [example](https://github.com/terraform-providers/terraform-provider-oci/blob/master/examples/database/adb/autonomous_database_wallet.tf) for more details.
diff --git a/website/docs/d/database_autonomous_database_wallet.html.markdown b/website/docs/d/database_autonomous_database_wallet.html.markdown
@@ -28,10 +28,12 @@ The following arguments are supported:
 
 * `autonomous_database_id` - (Required) The database [OCID](https://docs.cloud.oracle.com/iaas/Content/General/Concepts/identifiers.htm).
 * `password` - (Required) The password to encrypt the keys inside the wallet. The password must be at least 8 characters long and must include at least 1 letter and either 1 numeric character or 1 special character.
-
+* `base64_encode_content` - (Optional) Encodes the downloaded zipped wallet in base64. It is recommended to set this to `true` to avoid corrupting the zip file in Terraform state. The default value is `false` to preserve backwards compatibility with Terraform v0.11 configurations.
 
 ## Attributes Reference
 
 The following attributes are exported:
 
-* `content` - content of the downloaded zipped wallet for the Autonomous Database
+* `content` - content of the downloaded zipped wallet for the Autonomous Database. If `base64_encode_content` is set to `true`, then this content will be base64 encoded.
+If passing the base64 encoded content to a `local_file` resource, please use the `content_base64` attribute of the `local_file` resource.
+See this [example](https://github.com/terraform-providers/terraform-provider-oci/blob/master/examples/database/adb/autonomous_data_warehouse_wallet.tf) for more details.
diff --git a/website/docs/d/object_storage_object.html.markdown b/website/docs/d/object_storage_object.html.markdown
@@ -31,6 +31,8 @@ The following arguments are supported:
 * `namespace` - (Required) The Object Storage namespace used for the request.
 * `object` - (Required) The name of the object. Avoid entering confidential information. Example: `test/object1.log` 
 * `content_length_limit` - (Optional) The limit of the content length of the object body to download from the object store. The default is 1Mb.
+* `base64_encode_content` - (Optional) Encodes the downloaded content in base64. It is recommended to set this to `true` for binary content to avoid corrupting the zip file in Terraform state. The default value is `false` to preserve backwards compatibility with Terraform v0.11 configurations.
+If passing the base64 encoded content to a `local_file` resource, please use the `content_base64` attribute of the `local_file` resource.
 
 
 ## Attributes Reference

Original file line number	Diff line number	Diff line change
`@@ -6,14 +6,14 @@ resource "random_string" "autonomous_data_warehouse_wallet_password" {`
`6`	`6`	`}`
`7`	`7`
`8`	`8`	`data "oci_database_autonomous_database_wallet" "autonomous_data_warehouse_wallet" {`
`9`		`- #Required`
`10`	`9`	`autonomous_database_id = "${oci_database_autonomous_database.autonomous_data_warehouse.id}"`
`11`	`10`	`password = "${random_string.autonomous_data_warehouse_wallet_password.result}"`
	`11`	`+ base64_encode_content = "true"`
`12`	`12`	`}`
`13`	`13`
`14`	`14`	`resource "local_file" "autonomous_data_warehouse_wallet_file" {`
`15`		`- content = "${data.oci_database_autonomous_database_wallet.autonomous_data_warehouse_wallet.content}"`
`16`		`- filename = "${path.module}/autonomous_data_warehouse_wallet.zip"`
	`15`	`+ content_base64 = "${data.oci_database_autonomous_database_wallet.autonomous_data_warehouse_wallet.content}"`
	`16`	`+ filename = "${path.module}/autonomous_data_warehouse_wallet.zip"`
`17`	`17`	`}`
`18`	`18`
`19`	`19`	`output "autonomous_data_warehouse_wallet_password" {`
Original file line number	Diff line number	Diff line change
`@@ -7,3 +7,7 @@ provider "oci" {`
`7`	`7`	`private_key_path = "${var.private_key_path}"`
`8`	`8`	`region = "${var.region}"`
`9`	`9`	`}`
	`10`	`+`
	`11`	`+provider "local" {`
	`12`	`+ version = ">=1.3.0" # Need this version of the local provider to support base64 encoded inputs`
	`13`	`+}`