SWIFT-926 Enable Extended JSON Corpus Tests (#33)

Also: SWIFT-939 Update spec tests to include new timestamp test

Author: nspektor

Sources/BSON/BSONBinary.swift

@@ -208,7 +208,7 @@ extension BSONBinary: BSONValue {
         [
             "$binary": [
                 "base64": .string(Data(self.data.readableBytesView).base64EncodedString()),
-                "subType": .string(String(self.subtype.rawValue, radix: 16))
+                "subType": .string(String(format: "%02x", self.subtype.rawValue))
             ]
         ]
     }

Sources/BSON/Date+BSONValue.swift

@@ -33,7 +33,12 @@ extension Date: BSONValue {
             self = Date(msSinceEpoch: int)
         case let .string(s):
             // relaxed extended JSON
-            guard let date = ExtendedJSONDecoder.extJSONDateFormatter.date(from: s) else {
+            // If fractional seconds are omitted in the input (no decimal point "."),
+            // formatter should only account for seconds, otherwise formatter should take milliseconds into account
+            let formatter = s.contains(".")
+                ? ExtendedJSONDecoder.extJSONDateFormatterMilliseconds
+                : ExtendedJSONDecoder.extJSONDateFormatterSeconds
+            guard let date = formatter.date(from: s) else {
                 throw DecodingError._extendedJSONError(
                     keyPath: keyPath,
                     debugDescription: "Expected \(s) to be an ISO-8601 Internet Date/Time Format" +
@@ -42,7 +47,13 @@ extension Date: BSONValue {
             }
             self = date
         default:
-            return nil
+            throw DecodingError._extendedJSONError(
+                keyPath: keyPath,
+                debugDescription: "Expected \(value) to be canonical extended JSON representing a " +
+                    "64-bit signed integer giving millisecs relative to the epoch, as a string OR " +
+                    "relaxed extended JSON representing a ISO-8601 Internet Date/Time Format with " +
+                    "maximum time precision of milliseconds as a string."
+            )
         }
     }
 
@@ -52,7 +63,12 @@ extension Date: BSONValue {
         // relaxed extended json depending on if the date is between 1970 and 9999
         // 1970 is 0 milliseconds since epoch, and 10,000 is 253,402,300,800,000.
         if self.msSinceEpoch >= 0 && self.msSinceEpoch < 253_402_300_800_000 {
-            let date = ExtendedJSONDecoder.extJSONDateFormatter.string(from: self)
+            // Fractional seconds SHOULD have exactly 3 decimal places if the fractional part is non-zero.
+            // Otherwise, fractional seconds SHOULD be omitted if zero.
+            let formatter = self.timeIntervalSince1970.truncatingRemainder(dividingBy: 1) == 0
+                ? ExtendedJSONDecoder.extJSONDateFormatterSeconds
+                : ExtendedJSONDecoder.extJSONDateFormatterMilliseconds
+            let date = formatter.string(from: self)
             return ["$date": .string(date)]
         } else {
             return self.toCanonicalExtendedJSON()

Sources/BSON/Double+BSONValue.swift

@@ -41,14 +41,31 @@ extension Double: BSONValue {
         }
     }
 
+    /// Helper function to make sure ExtendedJSON formatting matches Corpus Tests
+    internal func formatForExtendedJSON() -> String {
+        if self.isNaN {
+            return "NaN"
+        } else if self == Double.infinity {
+            return "Infinity"
+        } else if self == -Double.infinity {
+            return "-Infinity"
+        } else {
+            return String(describing: self).uppercased()
+        }
+    }
+
     /// Converts this `Double` to a corresponding `JSON` in relaxed extendedJSON format.
-    func toRelaxedExtendedJSON() -> JSON {
-        .number(self)
+    internal func toRelaxedExtendedJSON() -> JSON {
+        if self.isInfinite || self.isNaN {
+            return self.toCanonicalExtendedJSON()
+        } else {
+            return .number(self)
+        }
     }
 
     /// Converts this `Double` to a corresponding `JSON` in canonical extendedJSON format.
-    func toCanonicalExtendedJSON() -> JSON {
-        ["$numberDouble": .string(String(describing: self))]
+    internal func toCanonicalExtendedJSON() -> JSON {
+        ["$numberDouble": .string(self.formatForExtendedJSON())]
     }
 
     internal static var bsonType: BSONType { .double }

Sources/BSON/ExtendedJSONDecoder.swift

@@ -1,7 +1,13 @@
 import Foundation
 /// `ExtendedJSONDecoder` facilitates the decoding of ExtendedJSON into `Decodable` values.
 public class ExtendedJSONDecoder {
-    internal static var extJSONDateFormatter: ISO8601DateFormatter = {
+    internal static var extJSONDateFormatterSeconds: ISO8601DateFormatter = {
+        let formatter = ISO8601DateFormatter()
+        formatter.formatOptions = [.withInternetDateTime]
+        return formatter
+    }()
+
+    internal static var extJSONDateFormatterMilliseconds: ISO8601DateFormatter = {
         let formatter = ISO8601DateFormatter()
         formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds]
         return formatter

Tests/BSONTests/BSONCorpusTests.swift

@@ -98,19 +98,53 @@ final class BSONCorpusTests: BSONTestCase {
 
     // swiftlint:disable:next cyclomatic_complexity
     func testBSONCorpus() throws {
+        let SKIPPED_CORPUS_TESTS = [
+            "Decimal128":
+                [
+                    // TODO: SWIFT-962
+                    "Exact rounding",
+                    "[dqbsr531] negatives (Rounded)",
+                    "[dqbsr431] check rounding modes heeded (Rounded)",
+                    "OK2",
+                    // TODO: SWIFT-965
+                    "[decq438] clamped zeros... (Clamped)",
+                    "[decq418] clamped zeros... (Clamped)"
+                ],
+            "Array":
+                [
+                    // TODO: SWIFT-963
+                    "Multi Element Array with duplicate indexes",
+                    "Single Element Array with index set incorrectly to empty string",
+                    "Single Element Array with index set incorrectly to ab"
+                ],
+            "Top-level document validity": [
+                "Bad DBRef (ref is number, not string)",
+                "Bad DBRef (db is number, not string)"
+            ]
+        ]
+
+        let shouldSkip = { testFileDesc, testDesc in
+            SKIPPED_CORPUS_TESTS[testFileDesc]?.contains { $0 == testDesc } == true
+        }
+
+        let decoder = ExtendedJSONDecoder()
+
         for (_, testFile) in try retrieveSpecTestFiles(specName: "bson-corpus", asType: BSONCorpusTestFile.self) {
             if let validityTests = testFile.valid {
                 for test in validityTests {
+                    guard !shouldSkip(testFile.description, test.description) else {
+                        continue
+                    }
                     guard let cBData = Data(hexString: test.canonicalBSON) else {
                         XCTFail("Unable to interpret canonical_bson as Data")
                         return
                     }
-                    // guard let cEJData = test.canonicalExtJSON.data(using: .utf8) else {
-                    //     XCTFail("Unable to interpret canonical_extjson as Data")
-                    //     return
-                    // }
+                    guard let cEJData = test.canonicalExtJSON.data(using: .utf8) else {
+                        XCTFail("Unable to interpret canonical_extjson as Data")
+                        return
+                    }
 
-                    // let lossy = test.lossy ?? false
+                    let lossy = test.lossy ?? false
 
                     // for cB input:
                     // native_to_bson( bson_to_native(cB) ) = cB
@@ -129,97 +163,93 @@ final class BSONCorpusTests: BSONTestCase {
                     let docFromNative = BSONDocument(fromArray: nativeFromDoc)
                     expect(docFromNative.toByteString()).to(equal(cBData.toByteString()))
 
-                    if testFile.description == "Decimal128" {
-                        // TODO: This should be tested by EXTJSON
-                        struct Decimal128CanonicalExtJSON: Codable {
-                            struct Value: Codable {
-                                enum CodingKeys: String, CodingKey {
-                                    case numberDecimal = "$numberDecimal"
-                                }
-
-                                var numberDecimal: String
-                            }
+                    // native_to_canonical_extended_json( bson_to_native(cB) ) = cEJ
+                    let canonicalEncoder = ExtendedJSONEncoder()
+                    canonicalEncoder.mode = .canonical
+                    expect(try canonicalEncoder.encode(docFromCB))
+                        .to(cleanEqual(test.canonicalExtJSON), description: test.description)
 
-                            var d: Value
-                        }
-                        let extjson = test.canonicalExtJSON.data(using: .ascii)!
-                        let jsonResult = try JSONDecoder().decode(Decimal128CanonicalExtJSON.self, from: extjson)
-                        let decimal128CorpusString = jsonResult.d.numberDecimal
+                    // native_to_relaxed_extended_json( bson_to_native(cB) ) = rEJ (if rEJ exists)
+                    let relaxedEncoder = ExtendedJSONEncoder() // default mode is .relaxed
+                    if let rEJ = test.relaxedExtJSON {
+                        expect(try relaxedEncoder.encode(docFromCB))
+                            .to(cleanEqual(rEJ), description: test.description)
+                    }
 
-                        let decimal128FromString = try BSONDecimal128(decimal128CorpusString)
-                        let decimal128FromBinary = docFromCB.d!.decimal128Value!
+                    // for cEJ input:
+                    // native_to_canonical_extended_json( json_to_native(cEJ) ) = cEJ
+                    expect(try canonicalEncoder.encode(try decoder.decode(BSONDocument.self, from: cEJData)))
+                        .to(cleanEqual(test.canonicalExtJSON), description: test.description)
 
-                        expect(decimal128FromString.description).to(equal(decimal128CorpusString))
-                        expect(decimal128FromBinary.description).to(equal(decimal128CorpusString))
+                    // native_to_bson( json_to_native(cEJ) ) = cB (unless lossy)
+                    if !lossy {
+                        expect(try decoder.decode(BSONDocument.self, from: cEJData))
+                            .to(sortedEqual(docFromCB), description: test.description)
                     }
 
-                    // native_to_canonical_extended_json( bson_to_native(cB) ) = cEJ
-                    // expect(docFromCB.canonicalExtendedJSON).to(cleanEqual(test.canonicalExtJSON))
+                    // for dB input (if it exists): (change to language native part)
+                    if let dB = test.degenerateBSON {
+                        guard let dBData = Data(hexString: dB) else {
+                            XCTFail("Unable to interpret degenerate_bson as Data")
+                            return
+                        }
 
-                    // native_to_relaxed_extended_json( bson_to_native(cB) ) = rEJ (if rEJ exists)
-                    // if let rEJ = test.relaxedExtJSON {
-                    //     expect(try Document(fromBSON: cBData).extendedJSON).to(cleanEqual(rEJ))
-                    // }
+                        let docFromDB = try BSONDocument(fromBSON: dBData)
 
-                    // for cEJ input:
-                    // native_to_canonical_extended_json( json_to_native(cEJ) ) = cEJ
-                    // expect(try Document(fromJSON: cEJData).canonicalExtendedJSON)
-                    //        .to(cleanEqual(test.canonicalExtJSON))
-
-                    // // native_to_bson( json_to_native(cEJ) ) = cB (unless lossy)
-                    // if !lossy {
-                    //     expect(try Document(fromJSON: cEJData).rawBSON).to(equal(cBData))
-                    // }
-
-                    // for dB input (if it exists):
-                    // if let dB = test.degenerateBSON {
-                    //     guard let dBData = Data(hexString: dB) else {
-                    //         XCTFail("Unable to interpret degenerate_bson as Data")
-                    //         return
-                    //     }
-
-                    //     // bson_to_canonical_extended_json(dB) = cEJ
-                    //     expect(try Document(fromBSON: dBData).canonicalExtendedJSON)
-                    //         .to(cleanEqual(test.canonicalExtJSON))
-
-                    //     // bson_to_relaxed_extended_json(dB) = rEJ (if rEJ exists)
-                    //     if let rEJ = test.relaxedExtJSON {
-                    //         expect(try Document(fromBSON: dBData).extendedJSON).to(cleanEqual(rEJ))
-                    //     }
-                    // }
+                        // SKIPPING: native_to_bson( bson_to_native(dB) ) = cB
+                        // We only validate the BSON bytes, we do not clean them up, so can't do this assertion
+                        // Degenerate BSON round trip tests will be added in SWIFT-964
 
-                    // for dEJ input (if it exists):
-                    // if let dEJ = test.degenerateExtJSON {
-                    //     // native_to_canonical_extended_json( json_to_native(dEJ) ) = cEJ
-                    //     expect(try Document(fromJSON: dEJ).canonicalExtendedJSON)
-                    //           .to(cleanEqual(test.canonicalExtJSON))
+                        // native_to_canonical_extended_json( bson_to_native(dB) ) = cEJ
+                        // (Not in spec yet, might be added in DRIVERS-1355)
+                        expect(try canonicalEncoder.encode(docFromDB))
+                            .to(cleanEqual(test.canonicalExtJSON))
 
-                    //     // native_to_bson( json_to_native(dEJ) ) = cB (unless lossy)
-                    //     if !lossy {
-                    //         expect(try Document(fromJSON: dEJ).rawBSON).to(equal(cBData))
-                    //     }
-                    // }
+                        // native_to_relaxed_extended_json( bson_to_native(dB) ) = rEJ (if rEJ exists)
+                        // (Not in spec yet, might be added in DRIVERS-1355)
+                        if let rEJ = test.relaxedExtJSON {
+                            expect(try relaxedEncoder.encode(docFromDB))
+                                .to(cleanEqual(rEJ), description: test.description)
+                        }
+                    }
+
+                    // for dEJ input (if it exists):
+                    if let dEJ = test.degenerateExtJSON, let dEJData = dEJ.data(using: .utf8) {
+                        // native_to_canonical_extended_json( json_to_native(dEJ) ) = cEJ
+                        expect(try canonicalEncoder.encode(try decoder.decode(BSONDocument.self, from: dEJData)))
+                            .to(cleanEqual(test.canonicalExtJSON), description: test.description)
+                        // native_to_bson( json_to_native(dEJ) ) = cB (unless lossy)
+                        if !lossy {
+                            try expect(try decoder.decode(BSONDocument.self, from: dEJData))
+                                .to(sortedEqual(BSONDocument(fromBSON: cBData)), description: test.description)
+                        }
+                    }
 
                     // for rEJ input (if it exists):
-                    // if let rEJ = test.relaxedExtJSON {
-                    //     // native_to_relaxed_extended_json( json_to_native(rEJ) ) = rEJ
-                    //     expect(try Document(fromJSON: rEJ).extendedJSON).to(cleanEqual(rEJ))
-                    // }
+                    if let rEJ = test.relaxedExtJSON, let rEJData = rEJ.data(using: .utf8) {
+                        // native_to_relaxed_extended_json( json_to_native(rEJ) ) = rEJ
+                        expect(try relaxedEncoder.encode(try decoder.decode(BSONDocument.self, from: rEJData)))
+                            .to(cleanEqual(rEJ), description: test.description)
+                    }
                 }
             }
 
             if let parseErrorTests = testFile.parseErrors {
-                continue // TODO: EXT JSON support required
                 for test in parseErrorTests {
+                    guard !shouldSkip(testFile.description, test.description) else {
+                        continue
+                    }
                     let description = "\(testFile.description)-\(test.description)"
-
                     switch BSONType(rawValue: UInt8(testFile.bsonType.dropFirst(2), radix: 16)!)! {
                     case .invalid: // "top level document" uses 0x00 for the bson type
-                        _ = ()
-                    // expect(try BSONDocument(fromJSON: test.string)).to(throwError(), description: description)
+                        guard let testData = test.string.data(using: .utf8) else {
+                            XCTFail("Unable to interpret canonical_bson as Data")
+                            return
+                        }
+                        expect(try decoder.decode(BSONDocument.self, from: testData))
+                            .to(throwError(errorType: DecodingError.self), description: description)
                     case .decimal128:
-                        _ = ()
-                    // expect(BSONDecimal128(test.string)).to(beNil(), description: description)
+                        continue // TODO: SWIFT-968
                     default:
                         throw TestError(
                             message: "\(description): parse error tests not implemented"
@@ -231,6 +261,9 @@ final class BSONCorpusTests: BSONTestCase {
 
             if let decodeErrors = testFile.decodeErrors {
                 for test in decodeErrors {
+                    guard !shouldSkip(testFile.description, test.description) else {
+                        continue
+                    }
                     let description = "\(testFile.description)-\(test.description)"
                     guard let data = Data(hexString: test.bson) else {
                         XCTFail("\(description): Unable to interpret bson as Data")