chore: update smithy version to 1.22.0 (#425)

## Motivation

awslabs/aws-sdk-swift#597 is failing due to unresolved shape smithy.api#Unit which was introduced in smithy-lang/smithy#980

## Changes
- Updated smithyVersion to 1.22.0.
- Update smithyGradleVersion to 0.6.0
- Fix issues with header list parsing. The parsing was always correct but the test setup was not, where expected result always assumed that the list is a date list. To support this, I took kotlin implementation of string list parsing and translated to Swift with test cases.
- JsonName trait now has no effect in AwsJson1_0 and AwsJson1_1, this change is covered at chore: update smithy version to 1.22.0 aws-sdk-swift#600
- serde change: in case where the top container is sparse map which contains a dense list as value, the dense list was considered as sparse. Therefore, added a way to track the member parent to fix the nullability.
- Sensitive trait fixes because now it can't target member directly.

## Result

awslabs/aws-sdk-swift#600

Author: ganeshnj

Packages/ClientRuntime/Sources/Networking/Http/UnknownHttpServiceError.swift

@@ -4,7 +4,7 @@
  */
 
 /// General Service Error structure used when exact error could not be deduced from the `HttpResponse`
-public struct UnknownHttpServiceError: HttpServiceError {
+public struct UnknownHttpServiceError: HttpServiceError, Swift.Equatable {
     public var _isThrottling: Bool = false
 
     public var _statusCode: HttpStatusCode?

Packages/ClientRuntime/Sources/Serialization/SerializationUtils/HeaderUtils.swift

@@ -5,49 +5,153 @@
 
 import Foundation
 
+fileprivate extension String {
+    func readNextQuoted(startIdx: String.Index, delim: Character = ",") throws -> (String.Index, String) {
+        // startIdx is start of the quoted value, there must be at least an ending quotation mark
+        if !(self.index(after: startIdx) < self.endIndex) {
+            throw HeaderDeserializationError.invalidStringHeaderList(value: self)
+        }
+        
+        // find first non-escaped quote or end of string
+        var endIdx = self.index(after: startIdx)
+        while endIdx < self.endIndex {
+            let char = self[endIdx]
+            if char == "\\" {
+                // skip escaped chars
+                endIdx = self.index(after: endIdx)
+            } else if char == "\"" {
+                break
+            }
+            endIdx = self.index(after: endIdx)
+        }
+        
+        let next = self[self.index(after: startIdx)..<endIdx]
+        
+        // consume trailing quote
+        if endIdx >= self.endIndex || self[endIdx] != "\"" {
+            throw HeaderDeserializationError.invalidStringHeaderList(value: self)
+        }
+        assert(endIdx < self.endIndex)
+        assert(self[endIdx] == "\"")
+        
+        endIdx = self.index(after: endIdx)
+        
+        // consume delim
+        while endIdx < self.endIndex {
+            let char = self[endIdx]
+            if char == " " || char == "\t" {
+                endIdx = self.index(after: endIdx)
+            } else if char == delim {
+                endIdx = self.index(after: endIdx)
+                break
+            } else {
+                throw HeaderDeserializationError.invalidStringHeaderList(value: self)
+            }
+        }
+        
+        let unescaped = next.replacingOccurrences(of: "\\\"", with: "\"")
+            .replacingOccurrences(of: "\\\\", with: "\\")
+        
+        return (endIdx, unescaped)
+    }
+    
+    func readNextUnquoted(startIdx: String.Index, delim: Character = ",") -> (String.Index, String) {
+        assert(startIdx < self.endIndex)
+        
+        var endIdx = startIdx
+        
+        while endIdx < self.endIndex && self[endIdx] != delim {
+            endIdx = self.index(after: endIdx)
+        }
+        
+        let next = self[startIdx..<endIdx]
+        if endIdx < self.endIndex && self[endIdx] == delim {
+            endIdx = self.index(after: endIdx)
+        }
+        
+        return (endIdx, next.trim())
+    }
+}
+
+// chars in an HTTP header value that require quotations
+private let QUOTABLE_HEADER_VALUE_CHARS = "\",()"
+
+public func quoteHeaderValue(_ value: String) -> String {
+    if value.trim().count != value.count || value.contains(where: { char1 in
+        QUOTABLE_HEADER_VALUE_CHARS.contains { char2 in
+            char1 == char2
+        }
+    }) {
+        let formatted = value.replacingOccurrences(of: "\\", with: "\\\\")
+            .replacingOccurrences(of: "\"", with: "\\\"")
+        return "\"\(formatted)\""
+    } else {
+        return value
+    }
+}
+
 /// Expands the compact Header Representation of List of any type except Dates
-public func splitHeaderListValues(_ value: String?) -> [String]? {
-    guard let value = value else { return nil}
-    return value.components(separatedBy: ",").map { $0.trim() }
+public func splitHeaderListValues(_ value: String?) throws -> [String]? {
+    guard let value = value else {
+        return nil
+    }
+    var results: [String] = []
+    var currIdx = value.startIndex
+    
+    while currIdx < value.endIndex {
+        let next: (idx: String.Index, str: String)
+        
+        switch value[currIdx] {
+        case " ", "\t":
+            currIdx = value.index(after: currIdx)
+            continue
+        case "\"":
+            next = try value.readNextQuoted(startIdx: currIdx)
+        default:
+            next = value.readNextUnquoted(startIdx: currIdx)
+        }
+        
+        currIdx = next.idx
+        results.append(next.str)
+    }
+    
+    return results
+    
 }
 
 /// Expands the compact HTTP Header Representation of List of Dates
 public func splitHttpDateHeaderListValues(_ value: String?) throws -> [String]? {
     guard let value = value else { return nil}
 
-    let separator = ","
-    let totalSeparators = value.components(separatedBy: separator).count - 1
-    if totalSeparators <= 1 {
+    let n = value.filter({$0 == ","}).count
+    
+    if n <= 1 {
         return [value]
-    } else if totalSeparators % 2 == 0 {
-        return splitHeaderListValues(value)
+    } else if n % 2 == 0 {
+        throw HeaderDeserializationError.invalidTimestampHeaderList(value: value)
     }
 
     var cnt = 0
     var splits: [String] = []
-    var start = 0
-    var startIdx = value.index(value.startIndex, offsetBy: start)
-
-    for i in 1...value.count {
-        let currIdx = value.index(value.startIndex, offsetBy: i-1)
-        if value[currIdx] == "," {
+    var startIdx = value.startIndex
+    
+    for i in value.indices[value.startIndex..<value.endIndex] {
+        if value[i] == "," {
             cnt += 1
         }
-
+        
         // split on every other ','
         if cnt > 1 {
-            startIdx = value.index(value.startIndex, offsetBy: start)
-            splits.append(String(value[startIdx..<currIdx]).trim())
-            start = i + 1
+            splits.append(value[startIdx..<i].trim())
+            startIdx = value.index(after: i)
             cnt = 0
         }
     }
-
-    if start < value.count {
-        startIdx = value.index(value.startIndex, offsetBy: start)
-        splits.append(String(value[startIdx..<value.endIndex]).trim())
+    
+    if startIdx < value.endIndex {
+        splits.append(value[startIdx...].trim())
     }
-
+    
     return splits
 }
 
@@ -64,19 +168,19 @@ extension HeaderDeserializationError: LocalizedError {
         switch self {
         case .invalidTimestampHeaderList(let value):
             return NSLocalizedString("Invalid HTTP Header List with Timestamps: \(value)",
-                comment: "Client Deserialization Error")
+                                     comment: "Client Deserialization Error")
         case .invalidTimestampHeader(let value):
             return NSLocalizedString("Invalid HTTP Header with Timestamp: \(value)",
-            comment: "Client Deserialization Error")
+                                     comment: "Client Deserialization Error")
         case .invalidBooleanHeaderList(let value):
             return NSLocalizedString("Invalid HTTP Header List with Booleans: \(value)",
-                comment: "Client Deserialization Error")
+                                     comment: "Client Deserialization Error")
         case .invalidNumbersHeaderList(let value):
             return NSLocalizedString("Invalid HTTP Header List with Booleans: \(value)",
-                comment: "Client Deserialization Error")
+                                     comment: "Client Deserialization Error")
         case .invalidStringHeaderList(let value):
             return NSLocalizedString("Invalid HTTP Header List with Strings: \(value)",
-                comment: "Client Deserialization Error")
+                                     comment: "Client Deserialization Error")
         }
     }
 }

Packages/ClientRuntime/Tests/SerializationTests/SerializationUtilsTests/HeaderUtilsTests.swift

@@ -9,15 +9,15 @@ import ClientRuntime
 class HeaderUtilsTests: XCTestCase {
 
     func testSplitHeaderListValues() {
-        guard let headerCollectionValues = splitHeaderListValues("1") else {
+        guard let headerCollectionValues = try! splitHeaderListValues("1") else {
             XCTFail("splitting header list values unexpectedly returned nil")
             return
         }
         XCTAssertEqual([1], headerCollectionValues.map { Int($0) })
-        XCTAssertEqual([1, 2, 3], splitHeaderListValues("1,2,3")?.map { Int($0) })
+        XCTAssertEqual([1, 2, 3], try! splitHeaderListValues("1,2,3")?.map { Int($0) })
         // Trim whitespaces in beginning and end of string components
-        XCTAssertEqual([1, 2, 3], splitHeaderListValues(" 1, 2, 3 ")?.map { Int($0) })
-        XCTAssertEqual([nil, 1], splitHeaderListValues(",1")?.map { Int($0) })
+        XCTAssertEqual([1, 2, 3], try! splitHeaderListValues(" 1, 2, 3 ")?.map { Int($0) })
+        XCTAssertEqual([nil, 1], try! splitHeaderListValues(",1")?.map { Int($0) })
     }
 
     func testSplitHttpDateHeaderListValues() {
@@ -31,7 +31,67 @@ class HeaderUtilsTests: XCTestCase {
         ]
 
         for (headerListString, headerList) in dateHeaderTransformations {
-            XCTAssertEqual(headerList, try? splitHttpDateHeaderListValues(headerListString))
+            XCTAssertEqual(headerList, try! splitHttpDateHeaderListValues(headerListString))
+        }
+        
+        XCTAssertThrowsError(try splitHttpDateHeaderListValues("Mon, 16 Dec 2019 23:48:18 GMT, , Tue, 17 Dec 2019 23:48:18 GMT")) { error in
+            XCTAssertEqual("Invalid HTTP Header List with Timestamps: Mon, 16 Dec 2019 23:48:18 GMT, , Tue, 17 Dec 2019 23:48:18 GMT", error.localizedDescription)
+        }
+    }
+    
+    func testSplitBoolList() {
+        XCTAssertEqual(["true", "false", "true", "true"], try! splitHeaderListValues("true,\"false\",true,\"true\""))
+    }
+    
+    func testSplitIntList() {
+        XCTAssertEqual(["1"], try! splitHeaderListValues("1"))
+        XCTAssertEqual(["1", "2", "3"], try! splitHeaderListValues("1,2,3"))
+        XCTAssertEqual(["1", "2", "3"], try! splitHeaderListValues("1,  2,  3"))
+        
+        // quoted
+        XCTAssertEqual(["1", "2", "3", "-4", "5"], try! splitHeaderListValues("1,\"2\",3,\"-4\",5"))
+    }
+    
+    func testSplitStringList() {
+        XCTAssertEqual(["foo"], try! splitHeaderListValues("foo"))
+        
+        // trailing space
+        XCTAssertEqual(["fooTrailing"], try! splitHeaderListValues("fooTrailing   "))
+        
+        // leading and trailing space
+        XCTAssertEqual(["  foo  "], try! splitHeaderListValues("\"  foo  \""))
+        
+        // ignore spaces between values
+        XCTAssertEqual(["foo", "bar"], try! splitHeaderListValues("foo  ,  bar"))
+        XCTAssertEqual(["foo", "bar"], try! splitHeaderListValues("\"foo\"  ,  \"bar\""))
+        
+        // comma in quotes
+        XCTAssertEqual(["foo,bar", "baz"], try! splitHeaderListValues("\"foo,bar\",baz"))
+        
+        // comm in quotes w/trailing space
+        XCTAssertEqual(["foo,bar", "baz"], try! splitHeaderListValues("\"foo,bar\",baz  "))
+        
+        // quote in quotes
+        XCTAssertEqual(["foo\",bar", "\"asdf\"", "baz"], try! splitHeaderListValues("\"foo\\\",bar\",\"\\\"asdf\\\"\",baz"))
+        
+        // quote in quote w/spaces
+        XCTAssertEqual(["foo\",bar", "\"asdf  \"", "baz"], try! splitHeaderListValues("\"foo\\\",bar\", \"\\\"asdf  \\\"\", baz"))
+        
+        // empty quotes
+        XCTAssertEqual(["", "baz"], try! splitHeaderListValues("\"\",baz"))
+        
+        // escaped slashes
+        XCTAssertEqual(["foo", "(foo\\bar)"], try! splitHeaderListValues("foo, \"(foo\\\\bar)\""))
+        
+        // empty
+        XCTAssertEqual(["", "1"], try! splitHeaderListValues(",1"))
+        
+        XCTAssertThrowsError(try splitHeaderListValues("foo, bar, \"baz")) { error in
+            XCTAssertEqual("Invalid HTTP Header List with Strings: foo, bar, \"baz", error.localizedDescription)
+        }
+        
+        XCTAssertThrowsError(try splitHeaderListValues("foo  ,  \"bar\"  \tf,baz")) { error in
+            XCTAssertEqual("Invalid HTTP Header List with Strings: foo  ,  \"bar\"  \tf,baz", error.localizedDescription)
         }
     }
 }

Packages/SmithyTestUtil/Sources/RequestTestUtil/HttpRequestTestBase.swift

@@ -57,13 +57,7 @@ open class HttpRequestTestBase: XCTestCase {
         if let headers = headers {
             for (headerName, headerValue) in headers {
                 let value = sanitizeStringForNonConformingValues(headerValue)
-                do {
-                    if let values = try splitHttpDateHeaderListValues(value) {
-                        builder.withHeader(name: headerName, values: values)
-                    }
-                } catch let err {
-                    XCTFail(err.localizedDescription)
-                }
+                builder.withHeader(name: headerName, value: value)
             }
         }
 
@@ -288,20 +282,20 @@ open class HttpRequestTestBase: XCTestCase {
             XCTFail("There are expected headers and no actual headers.")
             return
         }
-
-        for expectedHeader in expected.headers {
-            XCTAssertTrue(actual.exists(name: expectedHeader.name),
-                          "expected header \(expectedHeader.name) has no actual values")
-            guard let values = actual.values(for: expectedHeader.name) else {
-                XCTFail("actual values expected to not be null")
+        
+        expected.headers.forEach { header in
+            XCTAssertTrue(actual.exists(name: header.name))
+            
+            guard actual.values(for: header.name) != header.value else {
+                XCTAssertEqual(actual.values(for: header.name), header.value)
                 return
             }
 
-            XCTAssert(expectedHeader.value.containsSameElements(as: values),
-                      """
-                      expected header name value pair not equal: \(expectedHeader.name):
-                      \(expectedHeader.value); found: \(expectedHeader.name):\(values)
-                      """)
+            let actualValue = actual.values(for: header.name)?.joined(separator: ", ")
+            XCTAssertNotNil(actualValue)
+            
+            let expectedValue = header.value.joined(separator: ", ")
+            XCTAssertEqual(actualValue, expectedValue)
         }
     }
 

gradle.properties

@@ -3,8 +3,8 @@ kotlin.code.style=official
 # config
 
 # codegen
-smithyVersion=1.13.1
-smithyGradleVersion=0.5.3
+smithyVersion=1.22.0
+smithyGradleVersion=0.6.0
 
 # kotlin
 kotlinVersion=1.5.31

smithy-swift-codegen/src/main/kotlin/software/amazon/smithy/swift/codegen/CodegenVisitor.kt

@@ -155,9 +155,9 @@ class CodegenVisitor(context: PluginContext) : ShapeVisitor.Default<Void>() {
 
     override fun structureShape(shape: StructureShape): Void? {
         writers.useShapeWriter(shape) { writer: SwiftWriter -> StructureGenerator(model, symbolProvider, writer, shape, settings, protocolGenerator?.serviceErrorProtocolSymbol).render() }
-        if (shape.hasTrait<SensitiveTrait>() || shape.members().any { it.hasTrait<SensitiveTrait>() }) {
+        if (shape.hasTrait<SensitiveTrait>() || shape.members().any { it.hasTrait<SensitiveTrait>() || model.expectShape(it.target).hasTrait<SensitiveTrait>() }) {
             writers.useShapeExtensionWriter(shape, "CustomDebugStringConvertible") { writer: SwiftWriter ->
-                CustomDebugStringConvertibleGenerator(symbolProvider, writer, shape).render()
+                CustomDebugStringConvertibleGenerator(symbolProvider, writer, shape, model).render()
             }
         }
         return null