chore(net): Add plaintext to encrypted table migration example (#1976)

Author: rishav-karanjit

.github/workflows/ci_examples_net.yml

@@ -94,3 +94,4 @@ jobs:
         shell: bash
         run: |
           dotnet run
+          dotnet test

Examples/runtimes/net/Examples.csproj

@@ -11,6 +11,9 @@
   <ItemGroup>
     <ProjectReference Include="../../../DynamoDbEncryption/runtimes/net/DynamoDbEncryption.csproj" />
     <PackageReference Include="AWSSDK.SecurityToken" Version="3.7.202.5"/>
+    <PackageReference Include="xunit" Version="2.4.0" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
   </ItemGroup>
 
 </Project>

Examples/runtimes/net/src/TestUtils.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Linq;
+using System.Collections.Generic;
 using Amazon.DynamoDBv2.Model;
 
 public class TestUtils
@@ -84,4 +85,21 @@ public static void PrintAttributeValue(AttributeValue value)
         if (value.IsBOOLSet) Console.Write($"BOOL {value.BOOL}\n");
         Console.Write("UNKNOWN\n");
     }
+
+    // Helper method to clean up test items
+    public static async System.Threading.Tasks.Task CleanupItems(string tableName, string partitionKey, string sortKey)
+    {
+        var ddb = new Amazon.DynamoDBv2.AmazonDynamoDBClient();
+        var key = new Dictionary<string, AttributeValue>
+        {
+            ["partition_key"] = new AttributeValue { S = partitionKey },
+            ["sort_key"] = new AttributeValue { N = sortKey }
+        };
+        var deleteRequest = new DeleteItemRequest
+        {
+            TableName = tableName,
+            Key = key
+        };
+        await ddb.DeleteItemAsync(deleteRequest);
+    }
 }

Examples/runtimes/net/src/migration/PlaintextToAWSDBE/MigrationUtils.cs

@@ -0,0 +1,51 @@
+using System;
+using System.Collections.Generic;
+using Amazon.DynamoDBv2.Model;
+
+namespace Examples.migration.PlaintextToAWSDBE
+{
+    /*
+    Utility class for the PlaintextToAWSDBE migration examples.
+    This class contains shared functionality used by all migration steps.
+    */
+    public class MigrationUtils
+    {
+        // Common attribute values used across all migration steps
+        public static readonly string ENCRYPTED_AND_SIGNED_VALUE = "this will be encrypted and signed";
+        public static readonly string SIGN_ONLY_VALUE = "this will never be encrypted, but it will be signed";
+        public static readonly string DO_NOTHING_VALUE = "this will never be encrypted nor signed";
+
+        // Verify that a returned item matches the expected values
+        public static bool VerifyReturnedItem(GetItemResponse response, string partitionKeyValue, string sortKeyValue)
+        {
+            var item = response.Item;
+
+            if (!item.ContainsKey("partition_key") || item["partition_key"].S != partitionKeyValue)
+            {
+                throw new Exception($"partition_key mismatch: expected {partitionKeyValue}, got {(item.ContainsKey("partition_key") ? item["partition_key"].S : "null")}");
+            }
+
+            if (!item.ContainsKey("sort_key") || item["sort_key"].N != sortKeyValue)
+            {
+                throw new Exception($"sort_key mismatch: expected {sortKeyValue}, got {(item.ContainsKey("sort_key") ? item["sort_key"].N : "null")}");
+            }
+
+            if (!item.ContainsKey("attribute1") || item["attribute1"].S != ENCRYPTED_AND_SIGNED_VALUE)
+            {
+                throw new Exception($"attribute1 mismatch: expected {ENCRYPTED_AND_SIGNED_VALUE}, got {(item.ContainsKey("attribute1") ? item["attribute1"].S : "null")}");
+            }
+
+            if (!item.ContainsKey("attribute2") || item["attribute2"].S != SIGN_ONLY_VALUE)
+            {
+                throw new Exception($"attribute2 mismatch: expected {SIGN_ONLY_VALUE}, got {(item.ContainsKey("attribute2") ? item["attribute2"].S : "null")}");
+            }
+
+            if (!item.ContainsKey("attribute3") || item["attribute3"].S != DO_NOTHING_VALUE)
+            {
+                throw new Exception($"attribute3 mismatch: expected {DO_NOTHING_VALUE}, got {(item.ContainsKey("attribute3") ? item["attribute3"].S : "null")}");
+            }
+
+            return true;
+        }
+    }
+}

Examples/runtimes/net/src/migration/PlaintextToAWSDBE/README.md

@@ -0,0 +1,51 @@
+# Plaintext DynamoDB Table to AWS Database Encryption SDK Encrypted Table Migration
+
+This projects demonstrates the steps necessary
+to migrate to the AWS Database Encryption SDK for DynamoDb
+from a plaintext database.
+
+[Step 0](plaintext/step0.go) demonstrates the starting state for your system.
+
+## Step 1
+
+In Step 1, you update your system to do the following:
+
+- continue to read plaintext items
+- continue to write plaintext items
+- prepare to read encrypted items
+
+When you deploy changes in Step 1,
+you should not expect any behavior change in your system,
+and your dataset still consists of plaintext data.
+
+You must ensure that the changes in Step 1 make it to all your readers before you proceed to Step 2.
+
+## Step 2
+
+In Step 2, you update your system to do the following:
+
+- continue to read plaintext items
+- start writing encrypted items
+- continue to read encrypted items
+
+When you deploy changes in Step 2,
+you are introducing encrypted items to your system,
+and must make sure that all your readers are updated with the changes from Step 1.
+
+Before you move onto the next step, you will need to encrypt all plaintext items in your dataset.
+Once you have completed this step,
+while new items are being encrypted using the new format and will be authenticated on read,
+your system will still accept reading plaintext, unauthenticated items.
+In order to complete migration to a system where you always authenticate your items,
+you should prioritize moving on to Step 3.
+
+## Step 3
+
+Once all old items are encrypted,
+update your system to do the following:
+
+- continue to write encrypted items
+- continue to read encrypted items
+- do not accept reading plaintext items
+
+Once you have deployed these changes to your system, you have completed migration.

Examples/runtimes/net/src/migration/PlaintextToAWSDBE/awsdbe/Common.cs

@@ -0,0 +1,84 @@
+using System.Collections.Generic;
+using AWS.Cryptography.DbEncryptionSDK.DynamoDb;
+using AWS.Cryptography.DbEncryptionSDK.StructuredEncryption;
+using AWS.Cryptography.MaterialProviders;
+
+namespace Examples.migration.PlaintextToAWSDBE
+{
+    public static class Common
+    {
+        public static Dictionary<string, DynamoDbTableEncryptionConfig> CreateTableConfigs(string kmsKeyId, string ddbTableName, PlaintextOverride PlaintextOverride)
+        {
+            // Create a Keyring. This Keyring will be responsible for protecting the data keys that protect your data.
+            // For this example, we will create a AWS KMS Keyring with the AWS KMS Key we want to use.
+            // We will use the `CreateMrkMultiKeyring` method to create this keyring,
+            // as it will correctly handle both single region and Multi-Region KMS Keys.
+            var matProv = new MaterialProviders(new MaterialProvidersConfig());
+            var keyringInput = new CreateAwsKmsMrkMultiKeyringInput { Generator = kmsKeyId };
+            var kmsKeyring = matProv.CreateAwsKmsMrkMultiKeyring(keyringInput);
+
+            // Configure which attributes are encrypted and/or signed when writing new items.
+            // For each attribute that may exist on the items we plan to write to our DynamoDbTable,
+            // we must explicitly configure how they should be treated during item encryption:
+            //  - ENCRYPT_AND_SIGN: The attribute is encrypted and included in the signature
+            //  - SIGN_ONLY: The attribute not encrypted, but is still included in the signature
+            //  - DO_NOTHING: The attribute is not encrypted and not included in the signature
+            string partitionKeyName = "partition_key";
+            string sortKeyName = "sort_key";
+            var attributeActionsOnEncrypt = new Dictionary<string, CryptoAction>
+            {
+                [partitionKeyName] = CryptoAction.SIGN_ONLY,
+                [sortKeyName] = CryptoAction.SIGN_ONLY,
+                ["attribute1"] = CryptoAction.ENCRYPT_AND_SIGN,
+                ["attribute2"] = CryptoAction.SIGN_ONLY,
+                ["attribute3"] = CryptoAction.DO_NOTHING
+            };
+
+            // Configure which attributes we expect to be excluded in the signature
+            // when reading items. There are two options for configuring this:
+            //
+            //    - (Recommended) Configure `allowedUnsignedAttributesPrefix`:
+            //      When defining your DynamoDb schema and deciding on attribute names,
+            //      choose a distinguishing prefix (such as ":") for all attributes that
+            //      you do not want to include in the signature.
+            //      This has two main benefits:
+            //      - It is easier to reason about the security and authenticity of data within your item
+            //        when all unauthenticated data is easily distinguishable by their attribute name.
+            //      - If you need to add new unauthenticated attributes in the future,
+            //        you can easily make the corresponding update to your `attributeActionsOnEncrypt`
+            //        and immediately start writing to that new attribute, without
+            //        any other configuration update needed.
+            //      Once you configure this field, it is not safe to update it.
+            //
+            //    - Configure `allowedUnsignedAttributes`: You may also explicitly list
+            //      a set of attributes that should be considered unauthenticated when encountered
+            //      on read. Be careful if you use this configuration. Do not remove an attribute
+            //      name from this configuration, even if you are no longer writing with that attribute,
+            //      as old items may still include this attribute, and our configuration needs to know
+            //      to continue to exclude this attribute from the signature scope.
+            //      If you add new attribute names to this field, you must first deploy the update to this
+            //      field to all readers in your host fleet before deploying the update to start writing
+            //      with that new attribute.
+            //
+            //   For this example, we will explicitly list the attributes that are not signed.
+            var unsignedAttributes = new List<string> { "attribute3" };
+
+            // Create the DynamoDb Encryption configuration for the table we will be writing to.
+            var tableConfig = new DynamoDbTableEncryptionConfig
+            {
+                LogicalTableName = ddbTableName,
+                PartitionKeyName = partitionKeyName,
+                SortKeyName = sortKeyName,
+                AttributeActionsOnEncrypt = attributeActionsOnEncrypt,
+                Keyring = kmsKeyring,
+                AllowedUnsignedAttributes = unsignedAttributes,
+                PlaintextOverride = PlaintextOverride
+            };
+
+            return new Dictionary<string, DynamoDbTableEncryptionConfig>
+            {
+                [ddbTableName] = tableConfig
+            };
+        }
+    }
+}

Examples/runtimes/net/src/migration/PlaintextToAWSDBE/awsdbe/MigrationStep1.cs

@@ -0,0 +1,117 @@
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Amazon.DynamoDBv2;
+using Amazon.DynamoDBv2.Model;
+using System.Diagnostics;
+using System.Net;
+using AWS.Cryptography.DbEncryptionSDK.DynamoDb;
+using AWS.Cryptography.DbEncryptionSDK.StructuredEncryption;
+using AWS.Cryptography.MaterialProviders;
+using Examples.migration.PlaintextToAWSDBE;
+
+namespace Examples.migration.PlaintextToAWSDBE.awsdbe
+{
+    /*
+    Migration Step 1: This is the first step in the migration process from
+    plaintext to encrypted DynamoDB using the AWS Database Encryption SDK.
+    
+    In this example, we configure a DynamoDB Encryption client to do the following:
+    1. Write items only in plaintext
+    2. Read items in plaintext or, if the item is encrypted, decrypt with our encryption configuration
+    
+    While this step configures your client to be ready to start reading encrypted items,
+    we do not yet expect to be reading any encrypted items,
+    as our client still writes plaintext items.
+    Before you move on to step 2, ensure that these changes have successfully been deployed
+    to all of your readers.
+    
+    Running this example requires access to the DDB Table whose name
+    is provided in the function parameter.
+    This table must be configured with the following
+    primary key configuration:
+      - Partition key is named "partition_key" with type (S)
+      - Sort key is named "sort_key" with type (N)
+    */
+    public class MigrationStep1
+    {
+        public static async Task<bool> MigrationStep1Example(string kmsKeyId, string ddbTableName, string partitionKeyValue, string sortKeyWriteValue, string sortKeyReadValue)
+        {
+            // 1. Create table configurations
+            // In this of migration we will use PlaintextOverride.FORCE_PLAINTEXT_WRITE_ALLOW_PLAINTEXT_READ
+            // which means:
+            //     - Write: Items are forced to be written as plaintext.
+            //              Items may not be written as encrypted items.
+            //     - Read: Items are allowed to be read as plaintext.
+            //             Items are allowed to be read as encrypted items.
+            var tableConfigs = Common.CreateTableConfigs(kmsKeyId, ddbTableName, PlaintextOverride.FORCE_PLAINTEXT_WRITE_ALLOW_PLAINTEXT_READ);
+
+            // 1. Create a new AWS SDK DynamoDb client using the TableEncryptionConfigs
+            var ddb = new Client.DynamoDbClient(
+                new DynamoDbTablesEncryptionConfig { TableEncryptionConfigs = tableConfigs });
+
+            // 2. Put an item into our table using the above client.
+            //    This item will be stored in plaintext due to our PlaintextOverride configuration.
+            string partitionKeyName = "partition_key";
+            string sortKeyName = "sort_key";
+            string encryptedAndSignedValue = MigrationUtils.ENCRYPTED_AND_SIGNED_VALUE;
+            string signOnlyValue = MigrationUtils.SIGN_ONLY_VALUE;
+            string doNothingValue = MigrationUtils.DO_NOTHING_VALUE;
+            var item = new Dictionary<string, AttributeValue>
+            {
+                [partitionKeyName] = new AttributeValue { S = partitionKeyValue },
+                [sortKeyName] = new AttributeValue { N = sortKeyWriteValue },
+                ["attribute1"] = new AttributeValue { S = encryptedAndSignedValue },
+                ["attribute2"] = new AttributeValue { S = signOnlyValue },
+                ["attribute3"] = new AttributeValue { S = doNothingValue }
+            };
+
+            var putRequest = new PutItemRequest
+            {
+                TableName = ddbTableName,
+                Item = item
+            };
+
+            var putResponse = await ddb.PutItemAsync(putRequest);
+            Debug.Assert(putResponse.HttpStatusCode == HttpStatusCode.OK);
+
+            // 3. Get an item back from the table using the same client.
+            //    If this is an item written in plaintext (i.e. any item written
+            //    during Step 0 or 1), then the item will still be in plaintext.
+            //    If this is an item that was encrypted client-side (i.e. any item written
+            //    during Step 2 or after), then the item will be decrypted client-side
+            //    and surfaced as a plaintext item.
+            var key = new Dictionary<string, AttributeValue>
+            {
+                [partitionKeyName] = new AttributeValue { S = partitionKeyValue },
+                [sortKeyName] = new AttributeValue { N = sortKeyReadValue }
+            };
+
+            var getRequest = new GetItemRequest
+            {
+                TableName = ddbTableName,
+                Key = key,
+                // In this example we configure a strongly consistent read
+                // because we perform a read immediately after a write (for demonstrative purposes).
+                // By default, reads are only eventually consistent.
+                ConsistentRead = true
+            };
+
+            var getResponse = await ddb.GetItemAsync(getRequest);
+            Debug.Assert(getResponse.HttpStatusCode == HttpStatusCode.OK);
+
+            // 4. Verify we get the expected item back
+            if (getResponse.Item == null)
+            {
+                throw new Exception("No item found");
+            }
+
+            bool success = MigrationUtils.VerifyReturnedItem(getResponse, partitionKeyValue, sortKeyReadValue);
+            if (success)
+            {
+                Console.WriteLine("MigrationStep1 completed successfully");
+            }
+            return success;
+        }
+    }
+}