Skip to content

Commit 5c44631

Browse files
rebello95rebello95
authored
Add useDeterministicOrdering to JSONEncodingOptions (#1478)
* Add `useDeterministicOrdering` to `JSONEncodingOptions` Adds an option to ensure that JSON serialization is deterministic when serializing Protobuf `map` fields. This should be the only type that needs to be sorted, since individual fields are serialized in order by the generated code that invokes `try visitor.visit(...)` for each field. Fixes #1477
1 parent 514b0d8 commit 5c44631

File tree

4 files changed

+107
-39
lines changed

4 files changed

+107
-39
lines changed

‎Sources/SwiftProtobuf/JSONEncodingOptions.swift‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,5 +27,19 @@ public struct JSONEncodingOptions: Sendable {
2727
/// By default they are converted to JSON(lowerCamelCase) names.
2828
public var preserveProtoFieldNames: Bool = false
2929

30+
/// Whether to use deterministic ordering when serializing.
31+
///
32+
/// Note that the deterministic serialization is NOT canonical across languages.
33+
/// It is NOT guaranteed to remain stable over time. It is unstable across
34+
/// different builds with schema changes due to unknown fields. Users who need
35+
/// canonical serialization (e.g., persistent storage in a canonical form,
36+
/// fingerprinting, etc.) should define their own canonicalization specification
37+
/// and implement their own serializer rather than relying on this API.
38+
///
39+
/// If deterministic serialization is requested, map entries will be sorted
40+
/// by keys in lexographical order. This is an implementation detail
41+
/// and subject to change.
42+
public var useDeterministicOrdering: Bool = false
43+
3044
public init() {}
3145
}

‎Sources/SwiftProtobuf/JSONEncodingVisitor.swift‎

Lines changed: 31 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -358,39 +358,49 @@ internal struct JSONEncodingVisitor: Visitor {
358358
// Packed fields are handled the same as non-packed fields, so JSON just
359359
// relies on the default implementations in Visitor.swift
360360

361-
362-
363361
mutating func visitMapField<KeyType, ValueType: MapValueType>(fieldType: _ProtobufMap<KeyType, ValueType>.Type, value: _ProtobufMap<KeyType, ValueType>.BaseType, fieldNumber: Int) throws {
364-
try startField(for: fieldNumber)
365-
encoder.append(text: "{")
366-
var mapVisitor = JSONMapEncodingVisitor(encoder: JSONEncoder(), options: options)
367-
for (k,v) in value {
368-
try KeyType.visitSingular(value: k, fieldNumber: 1, with: &mapVisitor)
369-
try ValueType.visitSingular(value: v, fieldNumber: 2, with: &mapVisitor)
362+
try iterateAndEncode(map: value, fieldNumber: fieldNumber, isOrderedBefore: KeyType._lessThan) {
363+
(visitor: inout JSONMapEncodingVisitor, key, value) throws -> () in
364+
try KeyType.visitSingular(value: key, fieldNumber: 1, with: &visitor)
365+
try ValueType.visitSingular(value: value, fieldNumber: 2, with: &visitor)
370366
}
371-
encoder.append(utf8Bytes: mapVisitor.bytesResult)
372-
encoder.append(text: "}")
373367
}
374368

375369
mutating func visitMapField<KeyType, ValueType>(fieldType: _ProtobufEnumMap<KeyType, ValueType>.Type, value: _ProtobufEnumMap<KeyType, ValueType>.BaseType, fieldNumber: Int) throws where ValueType.RawValue == Int {
376-
try startField(for: fieldNumber)
377-
encoder.append(text: "{")
378-
var mapVisitor = JSONMapEncodingVisitor(encoder: JSONEncoder(), options: options)
379-
for (k, v) in value {
380-
try KeyType.visitSingular(value: k, fieldNumber: 1, with: &mapVisitor)
381-
try mapVisitor.visitSingularEnumField(value: v, fieldNumber: 2)
370+
try iterateAndEncode(map: value, fieldNumber: fieldNumber, isOrderedBefore: KeyType._lessThan) {
371+
(visitor: inout JSONMapEncodingVisitor, key, value) throws -> () in
372+
try KeyType.visitSingular(value: key, fieldNumber: 1, with: &visitor)
373+
try visitor.visitSingularEnumField(value: value, fieldNumber: 2)
382374
}
383-
encoder.append(utf8Bytes: mapVisitor.bytesResult)
384-
encoder.append(text: "}")
385375
}
386376

387377
mutating func visitMapField<KeyType, ValueType>(fieldType: _ProtobufMessageMap<KeyType, ValueType>.Type, value: _ProtobufMessageMap<KeyType, ValueType>.BaseType, fieldNumber: Int) throws {
378+
try iterateAndEncode(map: value, fieldNumber: fieldNumber, isOrderedBefore: KeyType._lessThan) {
379+
(visitor: inout JSONMapEncodingVisitor, key, value) throws -> () in
380+
try KeyType.visitSingular(value: key, fieldNumber: 1, with: &visitor)
381+
try visitor.visitSingularMessageField(value: value, fieldNumber: 2)
382+
}
383+
}
384+
385+
/// Helper to encapsulate the common structure of iterating over a map
386+
/// and encoding the keys and values.
387+
private mutating func iterateAndEncode<K, V>(
388+
map: Dictionary<K, V>,
389+
fieldNumber: Int,
390+
isOrderedBefore: (K, K) -> Bool,
391+
encode: (inout JSONMapEncodingVisitor, K, V) throws -> ()
392+
) throws {
388393
try startField(for: fieldNumber)
389394
encoder.append(text: "{")
390395
var mapVisitor = JSONMapEncodingVisitor(encoder: JSONEncoder(), options: options)
391-
for (k,v) in value {
392-
try KeyType.visitSingular(value: k, fieldNumber: 1, with: &mapVisitor)
393-
try mapVisitor.visitSingularMessageField(value: v, fieldNumber: 2)
396+
if options.useDeterministicOrdering {
397+
for (k,v) in map.sorted(by: { isOrderedBefore( $0.0, $1.0) }) {
398+
try encode(&mapVisitor, k, v)
399+
}
400+
} else {
401+
for (k,v) in map {
402+
try encode(&mapVisitor, k, v)
403+
}
394404
}
395405
encoder.append(utf8Bytes: mapVisitor.bytesResult)
396406
encoder.append(text: "}")

‎Sources/SwiftProtobuf/TextFormatEncodingVisitor.swift‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -503,16 +503,16 @@ internal struct TextFormatEncodingVisitor: Visitor {
503503
// fields (including proto3's default use of packed) without
504504
// introducing the baggage of a separate option.
505505

506-
private mutating func _visitPacked<T>(
507-
value: [T], fieldNumber: Int,
506+
private mutating func iterateAndEncode<T>(
507+
packedValue: [T], fieldNumber: Int,
508508
encode: (T, inout TextFormatEncoder) -> ()
509509
) throws {
510-
assert(!value.isEmpty)
510+
assert(!packedValue.isEmpty)
511511
emitFieldName(lookingUp: fieldNumber)
512512
encoder.startRegularField()
513513
var firstItem = true
514514
encoder.startArray()
515-
for v in value {
515+
for v in packedValue {
516516
if !firstItem {
517517
encoder.arraySeparator()
518518
}
@@ -524,42 +524,42 @@ internal struct TextFormatEncodingVisitor: Visitor {
524524
}
525525

526526
mutating func visitPackedFloatField(value: [Float], fieldNumber: Int) throws {
527-
try _visitPacked(value: value, fieldNumber: fieldNumber) {
527+
try iterateAndEncode(packedValue: value, fieldNumber: fieldNumber) {
528528
(v: Float, encoder: inout TextFormatEncoder) in
529529
encoder.putFloatValue(value: v)
530530
}
531531
}
532532

533533
mutating func visitPackedDoubleField(value: [Double], fieldNumber: Int) throws {
534-
try _visitPacked(value: value, fieldNumber: fieldNumber) {
534+
try iterateAndEncode(packedValue: value, fieldNumber: fieldNumber) {
535535
(v: Double, encoder: inout TextFormatEncoder) in
536536
encoder.putDoubleValue(value: v)
537537
}
538538
}
539539

540540
mutating func visitPackedInt32Field(value: [Int32], fieldNumber: Int) throws {
541-
try _visitPacked(value: value, fieldNumber: fieldNumber) {
541+
try iterateAndEncode(packedValue: value, fieldNumber: fieldNumber) {
542542
(v: Int32, encoder: inout TextFormatEncoder) in
543543
encoder.putInt64(value: Int64(v))
544544
}
545545
}
546546

547547
mutating func visitPackedInt64Field(value: [Int64], fieldNumber: Int) throws {
548-
try _visitPacked(value: value, fieldNumber: fieldNumber) {
548+
try iterateAndEncode(packedValue: value, fieldNumber: fieldNumber) {
549549
(v: Int64, encoder: inout TextFormatEncoder) in
550550
encoder.putInt64(value: v)
551551
}
552552
}
553553

554554
mutating func visitPackedUInt32Field(value: [UInt32], fieldNumber: Int) throws {
555-
try _visitPacked(value: value, fieldNumber: fieldNumber) {
555+
try iterateAndEncode(packedValue: value, fieldNumber: fieldNumber) {
556556
(v: UInt32, encoder: inout TextFormatEncoder) in
557557
encoder.putUInt64(value: UInt64(v))
558558
}
559559
}
560560

561561
mutating func visitPackedUInt64Field(value: [UInt64], fieldNumber: Int) throws {
562-
try _visitPacked(value: value, fieldNumber: fieldNumber) {
562+
try iterateAndEncode(packedValue: value, fieldNumber: fieldNumber) {
563563
(v: UInt64, encoder: inout TextFormatEncoder) in
564564
encoder.putUInt64(value: v)
565565
}
@@ -590,26 +590,26 @@ internal struct TextFormatEncodingVisitor: Visitor {
590590
}
591591

592592
mutating func visitPackedBoolField(value: [Bool], fieldNumber: Int) throws {
593-
try _visitPacked(value: value, fieldNumber: fieldNumber) {
593+
try iterateAndEncode(packedValue: value, fieldNumber: fieldNumber) {
594594
(v: Bool, encoder: inout TextFormatEncoder) in
595595
encoder.putBoolValue(value: v)
596596
}
597597
}
598598

599599
mutating func visitPackedEnumField<E: Enum>(value: [E], fieldNumber: Int) throws {
600-
try _visitPacked(value: value, fieldNumber: fieldNumber) {
600+
try iterateAndEncode(packedValue: value, fieldNumber: fieldNumber) {
601601
(v: E, encoder: inout TextFormatEncoder) in
602602
encoder.putEnumValue(value: v)
603603
}
604604
}
605605

606606
/// Helper to encapsulate the common structure of iterating over a map
607607
/// and encoding the keys and values.
608-
private mutating func _visitMap<K, V>(
608+
private mutating func iterateAndEncode<K, V>(
609609
map: Dictionary<K, V>,
610610
fieldNumber: Int,
611611
isOrderedBefore: (K, K) -> Bool,
612-
coder: (inout TextFormatEncodingVisitor, K, V) throws -> ()
612+
encode: (inout TextFormatEncodingVisitor, K, V) throws -> ()
613613
) throws {
614614
// Cache old visitor configuration
615615
let oldNameMap = self.nameMap
@@ -625,7 +625,7 @@ internal struct TextFormatEncodingVisitor: Visitor {
625625
self.nameResolver = mapNameResolver
626626
self.extensions = nil
627627

628-
try coder(&self, k, v)
628+
try encode(&self, k, v)
629629

630630
// Restore configuration before resuming containing message
631631
self.extensions = oldExtensions
@@ -641,7 +641,7 @@ internal struct TextFormatEncodingVisitor: Visitor {
641641
value: _ProtobufMap<KeyType, ValueType>.BaseType,
642642
fieldNumber: Int
643643
) throws {
644-
try _visitMap(map: value, fieldNumber: fieldNumber, isOrderedBefore: KeyType._lessThan) {
644+
try iterateAndEncode(map: value, fieldNumber: fieldNumber, isOrderedBefore: KeyType._lessThan) {
645645
(visitor: inout TextFormatEncodingVisitor, key, value) throws -> () in
646646
try KeyType.visitSingular(value: key, fieldNumber: 1, with: &visitor)
647647
try ValueType.visitSingular(value: value, fieldNumber: 2, with: &visitor)
@@ -653,7 +653,7 @@ internal struct TextFormatEncodingVisitor: Visitor {
653653
value: _ProtobufEnumMap<KeyType, ValueType>.BaseType,
654654
fieldNumber: Int
655655
) throws where ValueType.RawValue == Int {
656-
try _visitMap(map: value, fieldNumber: fieldNumber, isOrderedBefore: KeyType._lessThan) {
656+
try iterateAndEncode(map: value, fieldNumber: fieldNumber, isOrderedBefore: KeyType._lessThan) {
657657
(visitor: inout TextFormatEncodingVisitor, key, value) throws -> () in
658658
try KeyType.visitSingular(value: key, fieldNumber: 1, with: &visitor)
659659
try visitor.visitSingularEnumField(value: value, fieldNumber: 2)
@@ -665,7 +665,7 @@ internal struct TextFormatEncodingVisitor: Visitor {
665665
value: _ProtobufMessageMap<KeyType, ValueType>.BaseType,
666666
fieldNumber: Int
667667
) throws {
668-
try _visitMap(map: value, fieldNumber: fieldNumber, isOrderedBefore: KeyType._lessThan) {
668+
try iterateAndEncode(map: value, fieldNumber: fieldNumber, isOrderedBefore: KeyType._lessThan) {
669669
(visitor: inout TextFormatEncodingVisitor, key, value) throws -> () in
670670
try KeyType.visitSingular(value: key, fieldNumber: 1, with: &visitor)
671671
try visitor.visitSingularMessageField(value: value, fieldNumber: 2)

‎Tests/SwiftProtobufTests/Test_JSONEncodingOptions.swift‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,4 +272,48 @@ class Test_JSONEncodingOptions: XCTestCase {
272272
XCTAssertEqual(try msg7.jsonString(options: protoNames),
273273
"{\"@type\":\"type.googleapis.com/swift_proto_testing.TestAllTypes\",\"optional_nested_enum\":\"NEG\"}")
274274
}
275+
276+
func testUseDeterministicOrdering() {
277+
var options = JSONEncodingOptions()
278+
options.useDeterministicOrdering = true
279+
280+
let stringMap = SwiftProtoTesting_Message3.with {
281+
$0.mapStringString = [
282+
"b": "B",
283+
"a": "A",
284+
"0": "0",
285+
"UPPER": "v",
286+
"x": "X",
287+
]
288+
}
289+
XCTAssertEqual(
290+
try stringMap.jsonString(options: options),
291+
"{\"mapStringString\":{\"0\":\"0\",\"UPPER\":\"v\",\"a\":\"A\",\"b\":\"B\",\"x\":\"X\"}}"
292+
)
293+
294+
let messageMap = SwiftProtoTesting_Message3.with {
295+
$0.mapInt32Message = [
296+
5: .with { $0.optionalSint32 = 5 },
297+
1: .with { $0.optionalSint32 = 1 },
298+
3: .with { $0.optionalSint32 = 3 },
299+
]
300+
}
301+
XCTAssertEqual(
302+
try messageMap.jsonString(options: options),
303+
"{\"mapInt32Message\":{\"1\":{\"optionalSint32\":1},\"3\":{\"optionalSint32\":3},\"5\":{\"optionalSint32\":5}}}"
304+
)
305+
306+
let enumMap = SwiftProtoTesting_Message3.with {
307+
$0.mapInt32Enum = [
308+
5: .foo,
309+
3: .bar,
310+
0: .baz,
311+
1: .extra3,
312+
]
313+
}
314+
XCTAssertEqual(
315+
try enumMap.jsonString(options: options),
316+
"{\"mapInt32Enum\":{\"0\":\"BAZ\",\"1\":\"EXTRA_3\",\"3\":\"BAR\",\"5\":\"FOO\"}}"
317+
)
318+
}
275319
}

0 commit comments

Comments
 (0)