Merge remote-tracking branch 'origin/main-staging' into local-staged-release

aws-sdk-dotnet-automation · aws-sdk-dotnet-automation · commit ab105c8ef847 · 2025-03-13T21:49:51.000Z
diff --git a/generator/.DevConfigs/8c19427b-85bc-4459-9d7e-f003fe48b14d.json b/generator/.DevConfigs/8c19427b-85bc-4459-9d7e-f003fe48b14d.json
@@ -0,0 +1,11 @@
+{
+    "services": [
+        {
+            "serviceName": "DynamoDBv2",
+            "type": "patch",
+            "changeLogMessages": [
+                "Fixed an issue where for a Nullable DateTime property decorated with StoreAsEpochLong attribute, null value was incorrectly stored as -62135596800 in DynamoDB."
+            ]
+        }
+    ]
+}
diff --git a/sdk/src/Services/DynamoDBv2/Custom/AWSConfigs.DynamoDB.cs b/sdk/src/Services/DynamoDBv2/Custom/AWSConfigs.DynamoDB.cs
@@ -338,6 +338,11 @@ public partial class PropertyConfig
         /// </summary>
         public bool StoreAsEpoch { get; set; }
 
+        /// <summary>
+        /// Whether this property should be stored as epoch seconds integer (with support for dates AFTER 2038).
+        /// </summary>
+        public bool StoreAsEpochLong { get; set; }
+
         /// <summary>
         /// Initializes a PropertyConfig object for a specific property
         /// </summary>
@@ -356,6 +361,7 @@ internal PropertyConfig(PropertyConfigElement prop)
             Version = prop.Version.GetValueOrDefault(false);
             Converter = prop.Converter;
             StoreAsEpoch = prop.StoreAsEpoch.GetValueOrDefault(false);
+            StoreAsEpochLong = prop.StoreAsEpochLong.GetValueOrDefault(false);
         }
 #endif
     }
@@ -582,6 +588,7 @@ internal class PropertyConfigElement : SerializableConfigurationElement
         private const string versionKey = "version";
         private const string converterKey = "converter";
         private const string storeAsEpochKey = "storeAsEpoch";
+        private const string storeAsEpochLongKey = "storeAsEpochLong";
 
         [ConfigurationProperty(nameKey, IsRequired = true)]
         public string Name
@@ -625,6 +632,13 @@ public bool? StoreAsEpoch
             get { return (bool?)this[storeAsEpochKey]; }
             set { this[storeAsEpochKey] = value; }
         }
+
+        [ConfigurationProperty(storeAsEpochLongKey)]
+        public bool? StoreAsEpochLong
+        {
+            get { return (bool?)this[storeAsEpochLongKey]; }
+            set { this[storeAsEpochLongKey] = value; }
+        }
     }
 
     /// <summary>
diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/InternalModel.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/InternalModel.cs
@@ -932,6 +932,7 @@ private static void PopulateConfigFromMappings(ItemStorageConfig config, Diction
                     propertyStorage.IsIgnored = propertyConfig.Ignore;
                     propertyStorage.IsVersion = propertyConfig.Version;
                     propertyStorage.StoreAsEpoch = propertyConfig.StoreAsEpoch;
+                    propertyStorage.StoreAsEpochLong = propertyConfig.StoreAsEpochLong;
                 }
             }
         }
diff --git a/sdk/src/Services/DynamoDBv2/Custom/DocumentModel/Document.cs b/sdk/src/Services/DynamoDBv2/Custom/DocumentModel/Document.cs
@@ -315,6 +315,13 @@ internal static DynamoDBEntry DateTimeToEpochSecondsLong(DynamoDBEntry entry, st
         {
             try
             {
+                var primitiveValue = entry.AsPrimitive();
+
+                // entry.AsPrimitive() could return null for UnconvertedDynamoDBEntry. UnconvertedDynamoDBEntry will always have a value due to check in it's constructor. Hence, we can process it further for epoch conversion.
+                if (primitiveValue != null 
+                        && primitiveValue.Value == null)
+                    return entry;
+
                 var dateTime = entry.AsDateTime();
                 string epochSecondsAsString = AWSSDKUtils.ConvertToUnixEpochSecondsString(dateTime);
                 entry = new Primitive(epochSecondsAsString, saveAsNumeric: true);
diff --git a/sdk/src/Services/DynamoDBv2/Custom/DocumentModel/ExpectedState.cs b/sdk/src/Services/DynamoDBv2/Custom/DocumentModel/ExpectedState.cs
@@ -197,7 +197,7 @@ public Dictionary<string, ExpectedAttributeValue> ToExpectedAttributeMap()
         /// <returns></returns>
         public Dictionary<string, ExpectedAttributeValue> ToExpectedAttributeMap(DynamoDBEntryConversion conversion)
         {
-            return ToExpectedAttributeMap(conversion, epochAttributes: null, isEmptyStringValueEnabled: false);
+            return ToExpectedAttributeMap(conversion, epochAttributes: null, epochLongAttributes: null, isEmptyStringValueEnabled: false);
         }
 
         /// <summary>
@@ -207,11 +207,11 @@ public Dictionary<string, ExpectedAttributeValue> ToExpectedAttributeMap(DynamoD
         /// <returns></returns>
         public Dictionary<string, ExpectedAttributeValue> ToExpectedAttributeMap(Table table)
         {
-            return ToExpectedAttributeMap(table.Conversion, table.StoreAsEpoch, table.IsEmptyStringValueEnabled);
+            return ToExpectedAttributeMap(table.Conversion, table.StoreAsEpoch, table.StoreAsEpochLong, table.IsEmptyStringValueEnabled);
         }
 
         private Dictionary<string, ExpectedAttributeValue> ToExpectedAttributeMap(DynamoDBEntryConversion conversion,
-            IEnumerable<string> epochAttributes, bool isEmptyStringValueEnabled)
+            IEnumerable<string> epochAttributes, IEnumerable<string> epochLongAttributes, bool isEmptyStringValueEnabled)
         {
             Dictionary<string, ExpectedAttributeValue> ret = new Dictionary<string, ExpectedAttributeValue>();
 
@@ -226,6 +226,11 @@ private Dictionary<string, ExpectedAttributeValue> ToExpectedAttributeMap(Dynamo
                     var values = expectedValue.Values.Select(p => Document.DateTimeToEpochSeconds(p, attributeName)).ToList();
                     eav = ExpectedValue.ToExpectedAttributeValue(expectedValue.Exists, values, expectedValue.Comparison, conversion, isEmptyStringValueEnabled);
                 }
+                else if(epochLongAttributes != null && epochLongAttributes.Contains(attributeName))
+                {
+                    var values = expectedValue.Values.Select(p => Document.DateTimeToEpochSecondsLong(p, attributeName)).ToList();
+                    eav = ExpectedValue.ToExpectedAttributeValue(expectedValue.Exists, values, expectedValue.Comparison, conversion, isEmptyStringValueEnabled);
+                }
                 else
                 {
                     eav = expectedValue.ToExpectedAttributeValue(conversion, isEmptyStringValueEnabled);
diff --git a/sdk/src/Services/DynamoDBv2/Custom/DocumentModel/Expression.cs b/sdk/src/Services/DynamoDBv2/Custom/DocumentModel/Expression.cs
@@ -236,6 +236,9 @@ internal static Dictionary<string, AttributeValue> ConvertToAttributeValues(
                         if (table.StoreAsEpoch.Contains(attributeName))
                             entry = Document.DateTimeToEpochSeconds(entry, attributeName);
 
+                        if (table.StoreAsEpochLong.Contains(attributeName))
+                            entry = Document.DateTimeToEpochSecondsLong(entry, attributeName);
+
                         var attributeConversionConfig = new DynamoDBEntry.AttributeConversionConfig(table.Conversion, table.IsEmptyStringValueEnabled);
                         convertedValues[attributeName] = entry.ConvertToAttributeValue(attributeConversionConfig);
                     }
diff --git a/sdk/src/Services/DynamoDBv2/Custom/DocumentModel/Filter.cs b/sdk/src/Services/DynamoDBv2/Custom/DocumentModel/Filter.cs
@@ -115,7 +115,7 @@ public Condition ToCondition(DynamoDBEntryConversion conversion, bool shouldConv
             /// <returns></returns>
             public Condition ToCondition(DynamoDBEntryConversion conversion, bool shouldConvertToEpochSeconds, string attributeName)
             {
-                return ToCondition(conversion, shouldConvertToEpochSeconds, attributeName, false);
+                return ToCondition(conversion, shouldConvertToEpochSeconds, false, attributeName, false);
             }
 
             /// <summary>
@@ -128,6 +128,46 @@ public Condition ToCondition(DynamoDBEntryConversion conversion, bool shouldConv
             /// <returns></returns>
             public Condition ToCondition(DynamoDBEntryConversion conversion, bool shouldConvertToEpochSeconds,
                 string attributeName, bool isEmptyStringValueEnabled)
+            {
+                return ToCondition(conversion, shouldConvertToEpochSeconds, false, attributeName, isEmptyStringValueEnabled);
+            }
+
+            /// <summary>
+            /// Converts the FilterCondition to the Amazon.DynamoDBv2.Model.Condition object.
+            /// </summary>
+            /// <param name="conversion"></param>
+            /// <param name="shouldConvertToEpochSeconds"></param>
+            /// <param name="shouldConvertToEpochSecondsLong"></param>
+            /// <returns></returns>
+            public Condition ToCondition(DynamoDBEntryConversion conversion, bool shouldConvertToEpochSeconds, bool shouldConvertToEpochSecondsLong)
+            {
+                return ToCondition(conversion, shouldConvertToEpochSeconds: false, attributeName: null, isEmptyStringValueEnabled: shouldConvertToEpochSeconds);
+            }
+
+            /// <summary>
+            /// Converts the FilterCondition to the Amazon.DynamoDBv2.Model.Condition object.
+            /// </summary>
+            /// <param name="conversion"></param>
+            /// <param name="shouldConvertToEpochSeconds"></param>
+            /// <param name="shouldConvertToEpochSecondsLong"></param>
+            /// <param name="attributeName"></param>
+            /// <returns></returns>
+            public Condition ToCondition(DynamoDBEntryConversion conversion, bool shouldConvertToEpochSeconds, bool shouldConvertToEpochSecondsLong, string attributeName)
+            {
+                return ToCondition(conversion, shouldConvertToEpochSeconds, attributeName, false);
+            }
+
+            /// <summary>
+            /// Converts the FilterCondition to the Amazon.DynamoDBv2.Model.Condition object.
+            /// </summary>
+            /// <param name="conversion"></param>
+            /// <param name="shouldConvertToEpochSeconds"></param>
+            /// <param name="shouldConvertToEpochSecondsLong"></param>
+            /// <param name="attributeName"></param>
+            /// <param name="isEmptyStringValueEnabled"></param>
+            /// <returns></returns>
+            public Condition ToCondition(DynamoDBEntryConversion conversion, bool shouldConvertToEpochSeconds, bool shouldConvertToEpochSecondsLong,
+                string attributeName, bool isEmptyStringValueEnabled)
             {
                 var attributeValues = AttributeValues;
                 if (attributeValues == null)
@@ -138,7 +178,10 @@ public Condition ToCondition(DynamoDBEntryConversion conversion, bool shouldConv
                         var entry = DynamoDBEntries[i];
                         if (shouldConvertToEpochSeconds)
                             entry = Document.DateTimeToEpochSeconds(entry, attributeName);
-                        
+
+                        if (shouldConvertToEpochSecondsLong)
+                            entry = Document.DateTimeToEpochSecondsLong(entry, attributeName);
+
                         var attributeConversionConfig = new DynamoDBEntry.AttributeConversionConfig(conversion, isEmptyStringValueEnabled);
                         var attributeValue = entry.ConvertToAttributeValue(attributeConversionConfig);
                         attributeValues.Add(attributeValue);
@@ -192,7 +235,7 @@ public Dictionary<string, Condition> ToConditions()
         /// <returns>Map from attribute name to condition</returns>
         public Dictionary<string, Condition> ToConditions(DynamoDBEntryConversion conversion)
         {
-            return ToConditions(conversion, epochAttributes: null, isEmptyStringValueEnabled: false);
+            return ToConditions(conversion, epochAttributes: null, epochLongAttributes: null, isEmptyStringValueEnabled: false);
         }
 
         /// <summary>
@@ -203,7 +246,7 @@ public Dictionary<string, Condition> ToConditions(DynamoDBEntryConversion conver
         /// <returns>Map from attribute name to condition</returns>
         public Dictionary<string, Condition> ToConditions(DynamoDBEntryConversion conversion, bool isEmptyStringValueEnabled)
         {
-            return ToConditions(conversion, epochAttributes: null, isEmptyStringValueEnabled: isEmptyStringValueEnabled);
+            return ToConditions(conversion, epochAttributes: null, epochLongAttributes: null, isEmptyStringValueEnabled: isEmptyStringValueEnabled);
         }
 
         /// <summary>
@@ -213,19 +256,20 @@ public Dictionary<string, Condition> ToConditions(DynamoDBEntryConversion conver
         /// <returns>Map from attribute name to condition</returns>
         public Dictionary<string, Condition> ToConditions(Table table)
         {
-            return ToConditions(table.Conversion, table.StoreAsEpoch, table.IsEmptyStringValueEnabled);
+            return ToConditions(table.Conversion, table.StoreAsEpoch, table.StoreAsEpochLong, table.IsEmptyStringValueEnabled);
         }
 
         private Dictionary<string, Condition> ToConditions(DynamoDBEntryConversion conversion,
-            IEnumerable<string> epochAttributes, bool isEmptyStringValueEnabled)
+            IEnumerable<string> epochAttributes, IEnumerable<string> epochLongAttributes, bool isEmptyStringValueEnabled)
         {
             var dic = new Dictionary<string, Condition>();
             foreach (var kvp in Conditions)
             {
                 string name = kvp.Key;
                 FilterCondition fc = kvp.Value;
                 bool convertToEpochSeconds = epochAttributes != null && epochAttributes.Contains(name);
-                Condition condition = fc.ToCondition(conversion, convertToEpochSeconds, name, isEmptyStringValueEnabled);
+                bool convertToEpochSecondsLong = epochLongAttributes != null && epochLongAttributes.Contains(name);
+                Condition condition = fc.ToCondition(conversion, convertToEpochSeconds, convertToEpochSecondsLong, name, isEmptyStringValueEnabled);
 
                 dic[name] = condition;
             }
diff --git a/sdk/src/Services/DynamoDBv2/Custom/DocumentModel/Table.cs b/sdk/src/Services/DynamoDBv2/Custom/DocumentModel/Table.cs
@@ -258,6 +258,9 @@ internal Key MakeKey(Primitive hashKey, Primitive rangeKey)
             if (this.StoreAsEpoch.Contains(hashKeyName))
                 hashKey = KeyDateTimeToEpochSeconds(hashKey, hashKeyName);
 
+            if (this.StoreAsEpochLong.Contains(hashKeyName))
+                hashKey = KeyDateTimeToEpochSecondsLong(hashKey, hashKeyName);
+
             if (hashKeyDescription.Type != hashKey.Type)
                 throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture,
                     "Schema for table {0}, hash key {1}, is inconsistent with specified hash key value.", TableName, hashKeyName));
@@ -279,6 +282,9 @@ internal Key MakeKey(Primitive hashKey, Primitive rangeKey)
                 if (this.StoreAsEpoch.Contains(rangeKeyName))
                     rangeKey = KeyDateTimeToEpochSeconds(rangeKey, rangeKeyName);
 
+                if (this.StoreAsEpochLong.Contains(rangeKeyName))
+                    rangeKey = KeyDateTimeToEpochSecondsLong(rangeKey, rangeKeyName);
+
                 if (rangeKeyDescription.Type != rangeKey.Type)
                     throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture,
                         "Schema for table {0}, range key {1}, is inconsistent with specified range key value.", TableName, hashKeyName));
@@ -297,6 +303,13 @@ private static Primitive KeyDateTimeToEpochSeconds(Primitive key, string attribu
             return converted;
         }
 
+        private static Primitive KeyDateTimeToEpochSecondsLong(Primitive key, string attributeName)
+        {
+            DynamoDBEntry entry = Document.DateTimeToEpochSecondsLong(key, attributeName);
+            Primitive converted = entry as Primitive;
+            return converted;
+        }
+
         internal void UserAgentRequestEventHandlerSync(object sender, RequestEventArgs args)
         {
             UserAgentRequestEventHandler(sender, args, false);
@@ -469,7 +482,7 @@ public static Table LoadTable(IAmazonDynamoDB ddbClient, TableConfig config)
         /// <returns><see cref="DynamoDBEntryType"/> if it can be determined for the given property, otherwise an exception will be thrown</returns>
         private static DynamoDBEntryType GetPrimitiveEntryTypeForProperty(PropertyStorage property, DynamoDBFlatConfig flatConfig)
         {
-            if (property.StoreAsEpoch)
+            if (property.StoreAsEpoch || property.StoreAsEpochLong)
             {
                 return DynamoDBEntryType.Numeric;
             }
diff --git a/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs b/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs
diff --git a/sdk/test/Services/DynamoDBv2/IntegrationTests/TTLTests.cs b/sdk/test/Services/DynamoDBv2/IntegrationTests/TTLTests.cs

Original file line number	Diff line number	Diff line change
`@@ -932,6 +932,7 @@ private static void PopulateConfigFromMappings(ItemStorageConfig config, Diction`
`932`	`932`	`propertyStorage.IsIgnored = propertyConfig.Ignore;`
`933`	`933`	`propertyStorage.IsVersion = propertyConfig.Version;`
`934`	`934`	`propertyStorage.StoreAsEpoch = propertyConfig.StoreAsEpoch;`
	`935`	`+ propertyStorage.StoreAsEpochLong = propertyConfig.StoreAsEpochLong;`
`935`	`936`	`}`
`936`	`937`	`}`
`937`	`938`	`}`
Original file line number	Diff line number	Diff line change
`@@ -315,6 +315,13 @@ internal static DynamoDBEntry DateTimeToEpochSecondsLong(DynamoDBEntry entry, st`
`315`	`315`	`{`
`316`	`316`	`try`
`317`	`317`	`{`
	`318`	`+ var primitiveValue = entry.AsPrimitive();`
	`319`	`+`
	`320`	`+ // entry.AsPrimitive() could return null for UnconvertedDynamoDBEntry. UnconvertedDynamoDBEntry will always have a value due to check in it's constructor. Hence, we can process it further for epoch conversion.`
	`321`	`+ if (primitiveValue != null`
	`322`	`+ && primitiveValue.Value == null)`
	`323`	`+ return entry;`
	`324`	`+`
`318`	`325`	`var dateTime = entry.AsDateTime();`
`319`	`326`	`string epochSecondsAsString = AWSSDKUtils.ConvertToUnixEpochSecondsString(dateTime);`
`320`	`327`	`entry = new Primitive(epochSecondsAsString, saveAsNumeric: true);`
Original file line number	Diff line number	Diff line change
`@@ -197,7 +197,7 @@ public Dictionary<string, ExpectedAttributeValue> ToExpectedAttributeMap()`
`197`	`197`	`/// <returns></returns>`
`198`	`198`	`public Dictionary<string, ExpectedAttributeValue> ToExpectedAttributeMap(DynamoDBEntryConversion conversion)`
`199`	`199`	`{`
`200`		`- return ToExpectedAttributeMap(conversion, epochAttributes: null, isEmptyStringValueEnabled: false);`
	`200`	`+ return ToExpectedAttributeMap(conversion, epochAttributes: null, epochLongAttributes: null, isEmptyStringValueEnabled: false);`
`201`	`201`	`}`
`202`	`202`
`203`	`203`	`/// <summary>`
`@@ -207,11 +207,11 @@ public Dictionary<string, ExpectedAttributeValue> ToExpectedAttributeMap(DynamoD`
`207`	`207`	`/// <returns></returns>`
`208`	`208`	`public Dictionary<string, ExpectedAttributeValue> ToExpectedAttributeMap(Table table)`
`209`	`209`	`{`
`210`		`- return ToExpectedAttributeMap(table.Conversion, table.StoreAsEpoch, table.IsEmptyStringValueEnabled);`
	`210`	`+ return ToExpectedAttributeMap(table.Conversion, table.StoreAsEpoch, table.StoreAsEpochLong, table.IsEmptyStringValueEnabled);`
`211`	`211`	`}`
`212`	`212`
`213`	`213`	`private Dictionary<string, ExpectedAttributeValue> ToExpectedAttributeMap(DynamoDBEntryConversion conversion,`
`214`		`- IEnumerable<string> epochAttributes, bool isEmptyStringValueEnabled)`
	`214`	`+ IEnumerable<string> epochAttributes, IEnumerable<string> epochLongAttributes, bool isEmptyStringValueEnabled)`
`215`	`215`	`{`
`216`	`216`	`Dictionary<string, ExpectedAttributeValue> ret = new Dictionary<string, ExpectedAttributeValue>();`
`217`	`217`
`@@ -226,6 +226,11 @@ private Dictionary<string, ExpectedAttributeValue> ToExpectedAttributeMap(Dynamo`
`226`	`226`	`var values = expectedValue.Values.Select(p => Document.DateTimeToEpochSeconds(p, attributeName)).ToList();`
`227`	`227`	`eav = ExpectedValue.ToExpectedAttributeValue(expectedValue.Exists, values, expectedValue.Comparison, conversion, isEmptyStringValueEnabled);`
`228`	`228`	`}`
	`229`	`+ else if(epochLongAttributes != null && epochLongAttributes.Contains(attributeName))`
	`230`	`+ {`
	`231`	`+ var values = expectedValue.Values.Select(p => Document.DateTimeToEpochSecondsLong(p, attributeName)).ToList();`
	`232`	`+ eav = ExpectedValue.ToExpectedAttributeValue(expectedValue.Exists, values, expectedValue.Comparison, conversion, isEmptyStringValueEnabled);`
	`233`	`+ }`
`229`	`234`	`else`
`230`	`235`	`{`
`231`	`236`	`eav = expectedValue.ToExpectedAttributeValue(conversion, isEmptyStringValueEnabled);`
Original file line number	Diff line number	Diff line change
`@@ -236,6 +236,9 @@ internal static Dictionary<string, AttributeValue> ConvertToAttributeValues(`
`236`	`236`	`if (table.StoreAsEpoch.Contains(attributeName))`
`237`	`237`	`entry = Document.DateTimeToEpochSeconds(entry, attributeName);`
`238`	`238`
	`239`	`+ if (table.StoreAsEpochLong.Contains(attributeName))`
	`240`	`+ entry = Document.DateTimeToEpochSecondsLong(entry, attributeName);`
	`241`	`+`
`239`	`242`	`var attributeConversionConfig = new DynamoDBEntry.AttributeConversionConfig(table.Conversion, table.IsEmptyStringValueEnabled);`
`240`	`243`	`convertedValues[attributeName] = entry.ConvertToAttributeValue(attributeConversionConfig);`
`241`	`244`	`}`