Skip to content

Commit c5761f5

Browse files
AlexDainesAlexDaines
committed
Refactor JSON marshalling for sparse maps and update test cases
- Removed obsolete comments and ignored tests related to sparse JSON maps. - Enhanced the `JsonRPCStructureMarshaller` to handle nullable types and improved the processing of sparse maps. - Updated the `SparseJsonMapsRequestMarshaller` and `SparseJsonMapsResponseUnmarshaller` to ensure proper handling of null values in sparse maps. - Adjusted the test cases to reflect the changes in serialization logic for sparse maps.
1 parent ed61e17 commit c5761f5

File tree

8 files changed

+1085
-357
lines changed

8 files changed

+1085
-357
lines changed

‎generator/ProtocolTestsGenerator/smithy-dotnet-codegen/src/main/java/software/amazon/smithy/dotnet/codegen/customizations/ProtocolTestCustomizations.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,5 @@ private ProtocolTestCustomizations() {
100100
);
101101
public static final List<String> VNextTests = Arrays.asList(
102102
//These are the tests that are failing in v4 after updating to 1.54.0 and artifacts 1.0.3004.0. Each one needs to be investigated.
103-
"RestJsonSerializesSparseNullMapValues"
104103
);
105104
}

‎generator/ServiceClientGeneratorLib/Generators/Marshallers/JsonRPCStructureMarshaller.cs

Lines changed: 754 additions & 280 deletions
Large diffs are not rendered by default.

‎generator/ServiceClientGeneratorLib/Generators/Marshallers/JsonRPCStructureMarshaller.tt

Lines changed: 127 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,12 @@ namespace <#= this.Config.Namespace #>.Model.Internal.MarshallTransformat
4949
<#=new string(' ', level * 4)#> {
5050
<#=new string(' ', level * 4)#> context.Writer.WritePropertyName("<#=member.MarshallName#>");
5151
<#+
52-
string memberProperty = variableName + "." + member.PropertyName + (member.IsNullable ? ".Value" : string.Empty);
52+
string memberProperty = variableName + "." + member.PropertyName;
53+
// Only append .Value for nullable types that aren't already complex types
54+
if (member.IsNullable && !member.IsStructure && !member.IsList && !member.IsMap)
55+
{
56+
memberProperty += ".Value";
57+
}
5358
if(member.IsStructure || member.IsList || member.IsMap)
5459
{
5560
this.ProcessStructure(level + 1, variableName + "." + member.PropertyName, member.Shape);
@@ -153,19 +158,7 @@ namespace <#= this.Config.Namespace #>.Model.Internal.MarshallTransformat
153158
}
154159
else if(structure.IsMap)
155160
{
156-
#>
157-
<#=new string(' ', level * 4)#> context.Writer.WriteStartObject();
158-
<#=new string(' ', level * 4)#> foreach (var <#=flatVariableName#>Kvp in <#=variableName#>)
159-
<#=new string(' ', level * 4)#> {
160-
<#=new string(' ', level * 4)#> context.Writer.WritePropertyName(<#=flatVariableName#>Kvp.Key);
161-
<#=new string(' ', level * 4)#> var <#=flatVariableName#>Value = <#=flatVariableName#>Kvp.Value;
162-
163-
<#+
164-
ProcessStructure(level + 1, flatVariableName + "Value", structure.ValueShape);
165-
#>
166-
<#=new string(' ', level * 4)#> }
167-
<#=new string(' ', level * 4)#> context.Writer.WriteEndObject();
168-
<#+
161+
ProcessMap(level, variableName, structure);
169162
}
170163
else if(structure.IsDocument)
171164
{
@@ -207,8 +200,126 @@ namespace <#= this.Config.Namespace #>.Model.Internal.MarshallTransformat
207200

208201
}
209202
}
203+
204+
protected void ProcessMap(int level, string variableName, Shape structure)
205+
{
206+
string flatVariableName = variableName.Replace(".", "");
207+
#>
208+
<#=new string(' ', level * 4)#> context.Writer.WriteStartObject();
209+
<#=new string(' ', level * 4)#> foreach (var <#=flatVariableName#>Kvp in <#=variableName#>)
210+
<#=new string(' ', level * 4)#> {
211+
<#=new string(' ', level * 4)#> context.Writer.WritePropertyName(<#=flatVariableName#>Kvp.Key);
212+
<#=new string(' ', level * 4)#> var <#=flatVariableName#>Value = <#=flatVariableName#>Kvp.Value;
213+
214+
<#+ // Check for null values - only null checks for sparse maps as defined in customizations
215+
216+
bool isNullableMap = false;
217+
if (this.Structure != null)
218+
{
219+
var memberWithMap = this.Structure.Members.FirstOrDefault(m => m.PropertyName == variableName.Split('.').Last());
220+
isNullableMap = (memberWithMap != null && memberWithMap.UseNullable);
221+
}
222+
223+
if (isNullableMap)
224+
{
225+
#>
226+
<#=new string(' ', level * 4)#> if (<#=flatVariableName#>Value == null)
227+
<#=new string(' ', level * 4)#> {
228+
<#=new string(' ', level * 4)#> context.Writer.WriteNullValue();
229+
<#=new string(' ', level * 4)#> }
230+
<#=new string(' ', level * 4)#> else
231+
<#=new string(' ', level * 4)#> {
232+
<#+
233+
}
234+
235+
if(structure.ValueShape.IsBoolean)
236+
{
237+
if(isNullableMap)
238+
{
210239
#>
240+
<#=new string(' ', level * 4)#> context.Writer.WriteBooleanValue(<#=flatVariableName#>Value.Value);
211241
<#+
242+
}
243+
else
244+
{
245+
#>
246+
<#=new string(' ', level * 4)#> context.Writer.WriteBooleanValue(<#=flatVariableName#>Value);
247+
<#+
248+
}
249+
}
250+
else if(structure.ValueShape.IsInt || structure.ValueShape.IsLong || structure.ValueShape.IsFloat || structure.ValueShape.IsDouble)
251+
{
252+
if(isNullableMap)
253+
{
254+
#>
255+
<#=new string(' ', level * 4)#> context.Writer.WriteNumberValue(<#=flatVariableName#>Value.Value);
256+
<#+
257+
}
258+
else
259+
{
260+
#>
261+
<#=new string(' ', level * 4)#> context.Writer.WriteNumberValue(<#=flatVariableName#>Value);
262+
<#+
263+
}
264+
}
265+
else if(structure.ValueShape.IsTimeStamp)
266+
{
267+
if(isNullableMap)
268+
{
269+
if (structure.ValueShape.data[Shape.TimestampFormatKey] != null && !string.Equals(structure.ValueShape.data["timestampFormat"].ToString(), "unixTimestamp"))
270+
{
271+
#>
272+
<#=new string(' ', level * 4)#> context.Writer.WriteStringValue(<#=flatVariableName#>Value.Value);
273+
<#+
274+
}
275+
else
276+
{
277+
#>
278+
<#=new string(' ', level * 4)#> context.Writer.WriteNumberValue(Convert.ToInt64(StringUtils.FromDateTimeToUnixTimestamp(<#=flatVariableName#>Value.Value)));
279+
<#+
280+
}
281+
}
282+
else
283+
{
284+
if (structure.ValueShape.data[Shape.TimestampFormatKey] != null && !string.Equals(structure.ValueShape.data["timestampFormat"].ToString(), "unixTimestamp"))
285+
{
286+
#>
287+
<#=new string(' ', level * 4)#> context.Writer.WriteStringValue(<#=flatVariableName#>Value);
288+
<#+
289+
}
290+
else
291+
{
292+
#>
293+
<#=new string(' ', level * 4)#> context.Writer.WriteNumberValue(Convert.ToInt64(StringUtils.FromDateTimeToUnixTimestamp(<#=flatVariableName#>Value)));
294+
<#+
295+
}
296+
}
297+
}
298+
else
299+
{
300+
if(isNullableMap)
301+
{
302+
ProcessStructure(level + 2, flatVariableName + "Value", structure.ValueShape);
303+
}
304+
else
305+
{
306+
ProcessStructure(level + 1, flatVariableName + "Value", structure.ValueShape);
307+
}
308+
}
309+
310+
// Close the else block for nullable maps
311+
if (isNullableMap)
312+
{
313+
#>
314+
<#=new string(' ', level * 4)#> }
315+
<#+
316+
}
317+
#>
318+
<#=new string(' ', level * 4)#> }
319+
<#=new string(' ', level * 4)#> context.Writer.WriteEndObject();
320+
<#+
321+
}
322+
212323
protected void DetermineCustomMarshallerJsonWriteMethod(Member member, string memberProperty, int level)
213324
{
214325
if (String.Equals(member.CustomMarshallerTransformation,"Amazon.Util.AWSSDKUtils.ConvertToUnixEpochMilliseconds", StringComparison.OrdinalIgnoreCase))
@@ -224,8 +335,7 @@ namespace <#= this.Config.Namespace #>.Model.Internal.MarshallTransformat
224335
<#+
225336
}
226337
}
227-
#>
228-
<#+
338+
229339
/// https://smithy.io/2.0/aws/protocols/aws-restjson1-protocol.html#json-shape-serialization
230340
/// timestamps in json protocols use unixtimestamp if none is specified (by default)
231341
protected void DetermineNormalJsonWriteMethod(Shape shape, string memberProperty, int level)
@@ -256,6 +366,7 @@ namespace <#= this.Config.Namespace #>.Model.Internal.MarshallTransformat
256366
}
257367
else if (shape.IsInt || shape.IsLong || shape.IsFloat || shape.IsDouble)
258368
{
369+
// Simple numeric value
259370
#>
260371
<#=new string(' ', level * 4)#> context.Writer.WriteNumberValue(<#=memberProperty#>);
261372
<#+

‎sdk/test/ProtocolTests/Generated/RestJsonProtocol/dotnet-protocol-test-codegen/SparseJsonMaps.cs

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -86,12 +86,6 @@ public void RestJsonSparseJsonMapsRequest()
8686
/// <summary>
8787
/// Serializes JSON map values in sparse maps
8888
/// </summary>
89-
/*
90-
* This test either requires a breaking change and will be addressed
91-
* in V4, or has a backlog item to be fixed in the future. Please
92-
* refer to the VNextTests list to see which it is.
93-
* */
94-
[Ignore]
9589
[TestMethod]
9690
[TestCategory("ProtocolTest")]
9791
[TestCategory("RequestTest")]

‎sdk/test/Services/RestJsonProtocol/Custom/Model/Internal/MarshallTransformations/SparseJsonMapsRequestMarshaller.cs

Lines changed: 86 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,9 @@
3030
using Amazon.Runtime.Internal.Util;
3131
using System.Text.Json;
3232
using System.Buffers;
33+
#if !NETFRAMEWORK
3334
using ThirdParty.RuntimeBackports;
34-
35+
#endif
3536
#pragma warning disable CS0612,CS0618
3637
namespace Amazon.RestJsonProtocol.Model.Internal.MarshallTransformations
3738
{
@@ -70,97 +71,132 @@ public IRequest Marshall(SparseJsonMapsRequest publicRequest)
7071
using var memoryStream = new MemoryStream();
7172
using Utf8JsonWriter writer = new Utf8JsonWriter(memoryStream);
7273
#endif
74+
writer.WriteStartObject();
75+
var context = new JsonMarshallerContext(request, writer);
76+
if(publicRequest.IsSetSparseBooleanMap())
7377
{
74-
writer.WriteStartObject();
75-
var context = new JsonMarshallerContext(request, writer);
76-
if(publicRequest.IsSetSparseBooleanMap())
78+
context.Writer.WritePropertyName("sparseBooleanMap");
79+
context.Writer.WriteStartObject();
80+
foreach (var publicRequestSparseBooleanMapKvp in publicRequest.SparseBooleanMap)
7781
{
78-
context.Writer.WritePropertyName("sparseBooleanMap");
79-
context.Writer.WriteStartObject();
80-
foreach (var publicRequestSparseBooleanMapKvp in publicRequest.SparseBooleanMap)
81-
{
82-
context.Writer.WritePropertyName(publicRequestSparseBooleanMapKvp.Key);
83-
var publicRequestSparseBooleanMapValue = publicRequestSparseBooleanMapKvp.Value;
82+
context.Writer.WritePropertyName(publicRequestSparseBooleanMapKvp.Key);
83+
var publicRequestSparseBooleanMapValue = publicRequestSparseBooleanMapKvp.Value;
8484

85-
context.Writer.WriteBooleanValue(publicRequestSparseBooleanMapValue.Value);
85+
if (publicRequestSparseBooleanMapValue == null)
86+
{
87+
context.Writer.WriteNullValue();
88+
}
89+
else
90+
{
91+
context.Writer.WriteBooleanValue(publicRequestSparseBooleanMapValue.Value);
8692
}
87-
context.Writer.WriteEndObject();
8893
}
94+
context.Writer.WriteEndObject();
95+
}
8996

90-
if(publicRequest.IsSetSparseNumberMap())
97+
if(publicRequest.IsSetSparseNumberMap())
98+
{
99+
context.Writer.WritePropertyName("sparseNumberMap");
100+
context.Writer.WriteStartObject();
101+
foreach (var publicRequestSparseNumberMapKvp in publicRequest.SparseNumberMap)
91102
{
92-
context.Writer.WritePropertyName("sparseNumberMap");
93-
context.Writer.WriteStartObject();
94-
foreach (var publicRequestSparseNumberMapKvp in publicRequest.SparseNumberMap)
95-
{
96-
context.Writer.WritePropertyName(publicRequestSparseNumberMapKvp.Key);
97-
var publicRequestSparseNumberMapValue = publicRequestSparseNumberMapKvp.Value;
103+
context.Writer.WritePropertyName(publicRequestSparseNumberMapKvp.Key);
104+
var publicRequestSparseNumberMapValue = publicRequestSparseNumberMapKvp.Value;
98105

99-
context.Writer.WriteNumberValue(publicRequestSparseNumberMapValue.Value);
106+
if (publicRequestSparseNumberMapValue == null)
107+
{
108+
context.Writer.WriteNullValue();
109+
}
110+
else
111+
{
112+
context.Writer.WriteNumberValue(publicRequestSparseNumberMapValue.Value);
100113
}
101-
context.Writer.WriteEndObject();
102114
}
115+
context.Writer.WriteEndObject();
116+
}
103117

104-
if(publicRequest.IsSetSparseSetMap())
118+
if(publicRequest.IsSetSparseSetMap())
119+
{
120+
context.Writer.WritePropertyName("sparseSetMap");
121+
context.Writer.WriteStartObject();
122+
foreach (var publicRequestSparseSetMapKvp in publicRequest.SparseSetMap)
105123
{
106-
context.Writer.WritePropertyName("sparseSetMap");
107-
context.Writer.WriteStartObject();
108-
foreach (var publicRequestSparseSetMapKvp in publicRequest.SparseSetMap)
109-
{
110-
context.Writer.WritePropertyName(publicRequestSparseSetMapKvp.Key);
111-
var publicRequestSparseSetMapValue = publicRequestSparseSetMapKvp.Value;
124+
context.Writer.WritePropertyName(publicRequestSparseSetMapKvp.Key);
125+
var publicRequestSparseSetMapValue = publicRequestSparseSetMapKvp.Value;
112126

127+
if (publicRequestSparseSetMapValue == null)
128+
{
129+
context.Writer.WriteNullValue();
130+
}
131+
else
132+
{
113133
context.Writer.WriteStartArray();
114134
foreach(var publicRequestSparseSetMapValueListValue in publicRequestSparseSetMapValue)
115135
{
116136
context.Writer.WriteStringValue(publicRequestSparseSetMapValueListValue);
117137
}
118138
context.Writer.WriteEndArray();
119139
}
120-
context.Writer.WriteEndObject();
121140
}
141+
context.Writer.WriteEndObject();
142+
}
122143

123-
if(publicRequest.IsSetSparseStringMap())
144+
if(publicRequest.IsSetSparseStringMap())
145+
{
146+
context.Writer.WritePropertyName("sparseStringMap");
147+
context.Writer.WriteStartObject();
148+
foreach (var publicRequestSparseStringMapKvp in publicRequest.SparseStringMap)
124149
{
125-
context.Writer.WritePropertyName("sparseStringMap");
126-
context.Writer.WriteStartObject();
127-
foreach (var publicRequestSparseStringMapKvp in publicRequest.SparseStringMap)
128-
{
129-
context.Writer.WritePropertyName(publicRequestSparseStringMapKvp.Key);
130-
var publicRequestSparseStringMapValue = publicRequestSparseStringMapKvp.Value;
150+
context.Writer.WritePropertyName(publicRequestSparseStringMapKvp.Key);
151+
var publicRequestSparseStringMapValue = publicRequestSparseStringMapKvp.Value;
131152

153+
if (publicRequestSparseStringMapValue == null)
154+
{
155+
context.Writer.WriteNullValue();
156+
}
157+
else
158+
{
132159
context.Writer.WriteStringValue(publicRequestSparseStringMapValue);
133160
}
134-
context.Writer.WriteEndObject();
135161
}
162+
context.Writer.WriteEndObject();
163+
}
136164

137-
if(publicRequest.IsSetSparseStructMap())
165+
if(publicRequest.IsSetSparseStructMap())
166+
{
167+
context.Writer.WritePropertyName("sparseStructMap");
168+
context.Writer.WriteStartObject();
169+
foreach (var publicRequestSparseStructMapKvp in publicRequest.SparseStructMap)
138170
{
139-
context.Writer.WritePropertyName("sparseStructMap");
140-
context.Writer.WriteStartObject();
141-
foreach (var publicRequestSparseStructMapKvp in publicRequest.SparseStructMap)
142-
{
143-
context.Writer.WritePropertyName(publicRequestSparseStructMapKvp.Key);
144-
var publicRequestSparseStructMapValue = publicRequestSparseStructMapKvp.Value;
171+
context.Writer.WritePropertyName(publicRequestSparseStructMapKvp.Key);
172+
var publicRequestSparseStructMapValue = publicRequestSparseStructMapKvp.Value;
145173

174+
if (publicRequestSparseStructMapValue == null)
175+
{
176+
context.Writer.WriteNullValue();
177+
}
178+
else
179+
{
146180
context.Writer.WriteStartObject();
147181

148182
var marshaller = GreetingStructMarshaller.Instance;
149183
marshaller.Marshall(publicRequestSparseStructMapValue, context);
150184

151185
context.Writer.WriteEndObject();
152186
}
153-
context.Writer.WriteEndObject();
154187
}
188+
context.Writer.WriteEndObject();
189+
}
155190

156-
writer.WriteEndObject();
157-
writer.Flush();
191+
writer.WriteEndObject();
192+
writer.Flush();
193+
// ToArray() must be called here because aspects of sigv4 signing require a byte array
158194
#if !NETFRAMEWORK
159-
request.Content = arrayPoolBufferWriter.WrittenMemory.ToArray();
195+
request.Content = arrayPoolBufferWriter.WrittenMemory.ToArray();
160196
#else
161-
request.Content = memoryStream.ToArray();
197+
request.Content = memoryStream.ToArray();
162198
#endif
163-
}
199+
164200

165201

166202
return request;

0 commit comments

Comments
 (0)