fix: use unstable payload option for protojson codec

Author: ciricc

encoding/encoding.go

@@ -122,6 +122,25 @@ type PayloadCodec interface {
 	Codec
 }
 
+// UnstableSerializer is an interface that codecs can implement to indicate
+// they may produce non-deterministic output for the same input.
+type UnstableSerializer interface {
+	IsUnstable() bool
+}
+
+// IsUnstableSerialization returns true if the codec may produce non-deterministic output.
+// This is true for ProtoJSONCodec (protojson does not guarantee deterministic output)
+// and any codec implementing UnstableSerializer that returns true from IsUnstable().
+func IsUnstableSerialization(codec Codec) bool {
+	if _, ok := codec.(protoJSONCodec); ok {
+		return true
+	}
+	if unstable, ok := codec.(UnstableSerializer); ok {
+		return unstable.IsUnstable()
+	}
+	return false
+}
+
 // InputPayload is provided to Restate upon handler discovery, to teach the ingress how to validate incoming
 // request bodies.
 type InputPayload struct {
@@ -264,7 +283,10 @@ func (j protoJSONCodec) Unmarshal(data []byte, input any) (err error) {
 func (j protoJSONCodec) Marshal(output any) ([]byte, error) {
 	switch output := output.(type) {
 	case proto.Message:
-		return protojson.Marshal(output)
+		return protojson.MarshalOptions{
+			Indent:    "",
+			Multiline: false,
+		}.Marshal(output)
 	default:
 		return nil, fmt.Errorf("ProtoJSONCodec.Marshal called with a type that is not a proto.Message")
 	}

encoding/encoding_test.go

@@ -95,3 +95,23 @@ func TestGenerateJsonSchema(t *testing.T) {
 		})
 	}
 }
+
+func TestIsUnstableSerialization(t *testing.T) {
+	tests := []struct {
+		name     string
+		codec    Codec
+		expected bool
+	}{
+		{"JSONCodec", JSONCodec, false},
+		{"ProtoCodec", ProtoCodec, false},
+		{"ProtoJSONCodec", ProtoJSONCodec, true},
+		{"BinaryCodec", BinaryCodec, false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := IsUnstableSerialization(tt.codec)
+			require.Equal(t, tt.expected, result, "IsUnstableSerialization(%s) = %v, want %v", tt.name, result, tt.expected)
+		})
+	}
+}