Merge branch 'main' into ajewell/buckets

Author: ajewellamz

.github/workflows/ci_test_go.yml

@@ -140,8 +140,9 @@ jobs:
         run: |
           make test_go
 
-      - name: Test Examples
+      - name: Run and Test Examples
         if: matrix.library == 'DynamoDbEncryption'
         working-directory: ./Examples/runtimes/go
         run: |
           go run main.go
+          go test ./...

Examples/runtimes/go/migration/PlaintextToAWSDBE/README.md

@@ -0,0 +1,51 @@
+# Plaintext DynamoDB Table to AWS Database Encryption SDK Encrypted Table Migration
+
+This projects demonstrates the steps necessary
+to migrate to the AWS Database Encryption SDK for DynamoDb
+from a plaintext database.
+
+[Step 0](plaintext/step0.go) demonstrates the starting state for your system.
+
+## Step 1
+
+In Step 1, you update your system to do the following:
+
+- continue to read plaintext items
+- continue to write plaintext items
+- prepare to read encrypted items
+
+When you deploy changes in Step 1,
+you should not expect any behavior change in your system,
+and your dataset still consists of plaintext data.
+
+You must ensure that the changes in Step 1 make it to all your readers before you proceed to Step 2.
+
+## Step 2
+
+In Step 2, you update your system to do the following:
+
+- continue to read plaintext items
+- start writing encrypted items
+- continue to read encrypted items
+
+When you deploy changes in Step 2,
+you are introducing encrypted items to your system,
+and must make sure that all your readers are updated with the changes from Step 1.
+
+Before you move onto the next step, you will need to encrypt all plaintext items in your dataset.
+Once you have completed this step,
+while new items are being encrypted using the new format and will be authenticated on read,
+your system will still accept reading plaintext, unauthenticated items.
+In order to complete migration to a system where you always authenticate your items,
+you should prioritize moving on to Step 3.
+
+## Step 3
+
+Once all old items are encrypted,
+update your system to do the following:
+
+- continue to write encrypted items
+- continue to read encrypted items
+- do not accept reading plaintext items
+
+Once you have deployed these changes to your system, you have completed migration.

Examples/runtimes/go/migration/PlaintextToAWSDBE/awsdbe/common.go

@@ -0,0 +1,71 @@
+package awsdbe
+
+import (
+	"context"
+
+	mpl "github.com/aws/aws-cryptographic-material-providers-library/releases/go/mpl/awscryptographymaterialproviderssmithygenerated"
+	mpltypes "github.com/aws/aws-cryptographic-material-providers-library/releases/go/mpl/awscryptographymaterialproviderssmithygeneratedtypes"
+	dbesdkdynamodbencryptiontypes "github.com/aws/aws-database-encryption-sdk-dynamodb/releases/go/dynamodb-esdk/awscryptographydbencryptionsdkdynamodbsmithygeneratedtypes"
+	dbesdkstructuredencryptiontypes "github.com/aws/aws-database-encryption-sdk-dynamodb/releases/go/dynamodb-esdk/awscryptographydbencryptionsdkstructuredencryptionsmithygeneratedtypes"
+	"github.com/aws/aws-database-encryption-sdk-dynamodb/releases/go/dynamodb-esdk/examples/utils"
+)
+
+func configureTable(kmsKeyID, ddbTableName string, plaintextOverride dbesdkdynamodbencryptiontypes.PlaintextOverride) dbesdkdynamodbencryptiontypes.DynamoDbTablesEncryptionConfig {
+
+	// Create a Keyring. This Keyring will be responsible for protecting the data keys that protect your data.
+	// We will use the `CreateMrkMultiKeyring` method to create this keyring,
+	// as it will correctly handle both single region and Multi-Region KMS Keys.
+	matProv, err := mpl.NewClient(mpltypes.MaterialProvidersConfig{})
+	utils.HandleError(err)
+
+	keyringInput := mpltypes.CreateAwsKmsMrkMultiKeyringInput{
+		Generator: &kmsKeyID,
+	}
+	kmsKeyring, err := matProv.CreateAwsKmsMrkMultiKeyring(context.Background(), keyringInput)
+	utils.HandleError(err)
+
+	// Configure which attributes are encrypted and/or signed when writing new items.
+	// For each attribute that may exist on the items we plan to write to our DynamoDbTable,
+	// we must explicitly configure how they should be treated during item encryption:
+	//  - ENCRYPT_AND_SIGN: The attribute is encrypted and included in the signature
+	//  - SIGN_ONLY: The attribute not encrypted, but is still included in the signature
+	//  - DO_NOTHING: The attribute is not encrypted and not included in the signature
+	partitionKeyName := "partition_key"
+	sortKeyName := "sort_key"
+
+	attributeActions := map[string]dbesdkstructuredencryptiontypes.CryptoAction{
+		partitionKeyName: dbesdkstructuredencryptiontypes.CryptoActionSignOnly,
+		sortKeyName:      dbesdkstructuredencryptiontypes.CryptoActionSignOnly,
+		"attribute1":     dbesdkstructuredencryptiontypes.CryptoActionEncryptAndSign,
+		"attribute2":     dbesdkstructuredencryptiontypes.CryptoActionSignOnly,
+		"attribute3":     dbesdkstructuredencryptiontypes.CryptoActionDoNothing,
+	}
+
+	// Configure which attributes we expect to be excluded in the signature
+	// when reading items. This value represents all unsigned attributes
+	// across our entire dataset. If you ever want to add new unsigned attributes
+	// in the future, you must make an update to this field to all your readers
+	// before deploying any change to start writing that new data. It is not safe
+	// to remove attributes from this field.
+	unsignedAttributes := []string{"attribute3"}
+
+	// Create encryption configuration for table.
+	tableConfig := dbesdkdynamodbencryptiontypes.DynamoDbTableEncryptionConfig{
+		LogicalTableName:          ddbTableName,
+		PartitionKeyName:          partitionKeyName,
+		SortKeyName:               &sortKeyName,
+		AttributeActionsOnEncrypt: attributeActions,
+		Keyring:                   kmsKeyring,
+		AllowedUnsignedAttributes: unsignedAttributes,
+		PlaintextOverride:         &plaintextOverride,
+	}
+
+	tableConfigsMap := make(map[string]dbesdkdynamodbencryptiontypes.DynamoDbTableEncryptionConfig)
+	tableConfigsMap[ddbTableName] = tableConfig
+
+	listOfTableConfigs := dbesdkdynamodbencryptiontypes.DynamoDbTablesEncryptionConfig{
+		TableEncryptionConfigs: tableConfigsMap,
+	}
+
+	return listOfTableConfigs
+}

Examples/runtimes/go/migration/PlaintextToAWSDBE/awsdbe/step1.go

@@ -0,0 +1,124 @@
+package awsdbe
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/dynamodb"
+	"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
+
+	"github.com/aws/aws-database-encryption-sdk-dynamodb/releases/go/dynamodb-esdk/dbesdkmiddleware"
+	plaintexttoawsdbe "github.com/aws/aws-database-encryption-sdk-dynamodb/releases/go/dynamodb-esdk/examples/migration/PlaintextToAWSDBE"
+	"github.com/aws/aws-database-encryption-sdk-dynamodb/releases/go/dynamodb-esdk/examples/utils"
+
+	dbesdkdynamodbencryptiontypes "github.com/aws/aws-database-encryption-sdk-dynamodb/releases/go/dynamodb-esdk/awscryptographydbencryptionsdkdynamodbsmithygeneratedtypes"
+)
+
+/*
+Migration Step 1: This is an example demonstrating how to start using the
+AWS Database Encryption SDK with a pre-existing table with plaintext items.
+In this example, we configure a DynamoDb Encryption Interceptor to do the following:
+  - Write items only in plaintext
+  - Read items in plaintext or, if the item is encrypted, decrypt with our encryption configuration
+
+While this step configures your client to be ready to start reading encrypted items,
+we do not yet expect to be reading any encrypted items,
+as our client still writes plaintext items.
+Before you move on to step 2, ensure that these changes have successfully been deployed
+to all of your readers.
+
+Running this example requires access to the DDB Table whose name
+is provided in the function parameter.
+This table must be configured with the following
+primary key configuration:
+  - Partition key is named "partition_key" with type (S)
+  - Sort key is named "sort_key" with type (S)
+*/
+func MigrationStep1(kmsKeyID, ddbTableName, partitionKeyValue, sortKeyWriteValue, sortKeyReadValue string) error {
+	cfg, err := config.LoadDefaultConfig(context.TODO())
+	utils.HandleError(err)
+
+	// 1. Configure your Keyring, attribute actions,
+	// 	  allowedUnsignedAttributes, and encryption configuration for table.
+	//    This is common across all the steps.
+
+	// Note that while we still are not writing encrypted items,
+	// and our key will not be used to encrypt items in this example,
+	// our configuration specifies that we may read encrypted items,
+	// and we should expect to be able to decrypt and process any encrypted items.
+	// To that end, we must fully define our encryption configuration in
+	// this step.
+
+	// This `PlaintextOverrideForcePlaintextWriteAllowPlaintextRead` means:
+	//  - Write: Items are forced to be written as plaintext.
+	//           Items may not be written as encrypted items.
+	//  - Read: Items are allowed to be read as plaintext.
+	//          Items are allowed to be read as encrypted items.
+	listOfTableConfigs := configureTable(kmsKeyID, ddbTableName, dbesdkdynamodbencryptiontypes.PlaintextOverrideForcePlaintextWriteAllowPlaintextRead)
+
+	// 2. Create DynamoDB client with dbEsdkMiddleware
+	dbEsdkMiddleware, err := dbesdkmiddleware.NewDBEsdkMiddleware(listOfTableConfigs)
+	utils.HandleError(err)
+
+	ddb := dynamodb.NewFromConfig(cfg, dbEsdkMiddleware.CreateMiddleware())
+
+	// 3. Put an item into your table.
+	//    This item will be stored in plaintext.
+	encryptedAndSignedValue := "this will be encrypted and signed"
+	signOnlyValue := "this will never be encrypted, but it will be signed"
+	doNothingValue := "this will never be encrypted nor signed"
+	item := map[string]types.AttributeValue{
+		"partition_key": &types.AttributeValueMemberS{Value: partitionKeyValue},
+		"sort_key":      &types.AttributeValueMemberN{Value: sortKeyWriteValue},
+		"attribute1":    &types.AttributeValueMemberS{Value: encryptedAndSignedValue},
+		"attribute2":    &types.AttributeValueMemberS{Value: signOnlyValue},
+		"attribute3":    &types.AttributeValueMemberS{Value: doNothingValue},
+	}
+
+	putInput := dynamodb.PutItemInput{
+		TableName: &ddbTableName,
+		Item:      item,
+	}
+
+	_, err = ddb.PutItem(context.TODO(), &putInput)
+
+	// We return this error because we run test against the error.
+	// When used in production code, you can decide how you want to handle errors.
+	if err != nil {
+		return err
+	}
+
+	// 4. Get an item back from the table using the DynamoDb Client.
+	//    If this is an item written in plaintext (i.e. any item written
+	//    during Step 0 or 1), then the item will still be in plaintext.
+	//    If this is an item that was encrypted client-side (i.e. any item written
+	//    during Step 2 or after), then the item will be decrypted client-side
+	//    and surfaced as a plaintext item.
+	key := map[string]types.AttributeValue{
+		"partition_key": &types.AttributeValueMemberS{Value: partitionKeyValue},
+		"sort_key":      &types.AttributeValueMemberN{Value: sortKeyReadValue},
+	}
+
+	getInput := &dynamodb.GetItemInput{
+		TableName:      &ddbTableName,
+		Key:            key,
+		ConsistentRead: aws.Bool(true),
+	}
+
+	result, err := ddb.GetItem(context.TODO(), getInput)
+	// We return this error because we run test against the error.
+	// When used in production code, you can decide how you want to handle errors.
+	if err != nil {
+		return err
+	}
+
+	// Verify we got the expected item back
+	err = plaintexttoawsdbe.VerifyReturnedItem(result, partitionKeyValue, sortKeyReadValue, encryptedAndSignedValue, signOnlyValue, doNothingValue)
+	if err != nil {
+		return err
+	}
+	fmt.Println("MigrationStep1 completed successfully")
+	return nil
+}

Examples/runtimes/go/migration/PlaintextToAWSDBE/awsdbe/step1_test.go

@@ -0,0 +1,50 @@
+package awsdbe
+
+import (
+	"testing"
+
+	"github.com/aws/aws-database-encryption-sdk-dynamodb/releases/go/dynamodb-esdk/examples/migration/PlaintextToAWSDBE/plaintext"
+	"github.com/aws/aws-database-encryption-sdk-dynamodb/releases/go/dynamodb-esdk/examples/utils"
+	"github.com/google/uuid"
+)
+
+func TestMigrationStep1(t *testing.T) {
+	kmsKeyID := utils.KmsKeyID()
+	tableName := utils.DdbTableName()
+	partitionKey := uuid.New().String()
+	sortKeys := []string{"0", "1", "2", "3"}
+
+	// Successfully executes Step 1
+	err := MigrationStep1(kmsKeyID, tableName, partitionKey, sortKeys[1], sortKeys[1])
+	utils.HandleError(err)
+
+	// Given: Step 0 has succeeded
+	err = plaintext.MigrationStep0(tableName, partitionKey, sortKeys[0], sortKeys[0])
+	utils.HandleError(err)
+
+	// When: Execute Step 1 with sortReadValue=0, Then: Success (i.e. can read plaintext values)
+	err = MigrationStep1(kmsKeyID, tableName, partitionKey, sortKeys[1], sortKeys[0])
+	utils.HandleError(err)
+
+	// Given: Step 2 has succeeded
+	err = MigrationStep2(kmsKeyID, tableName, partitionKey, sortKeys[2], sortKeys[2])
+	utils.HandleError(err)
+
+	// When: Execute Step 1 with sortReadValue=2, Then: Success (i.e. can read encrypted values)
+	err = MigrationStep1(kmsKeyID, tableName, partitionKey, sortKeys[1], sortKeys[2])
+	utils.HandleError(err)
+
+	// Given: Step 3 has succeeded
+	err = MigrationStep3(kmsKeyID, tableName, partitionKey, sortKeys[3], sortKeys[3])
+	utils.HandleError(err)
+
+	// When: Execute Step 1 with sortReadValue=3, Then: Success (i.e. can read encrypted values)
+	err = MigrationStep1(kmsKeyID, tableName, partitionKey, sortKeys[1], sortKeys[3])
+	utils.HandleError(err)
+
+	// Cleanup
+	for _, sortKey := range sortKeys {
+		utils.DeleteItem(tableName, "partition_key", partitionKey, "sort_key", sortKey)
+	}
+
+}