Support non-deterministic codecs via PayloadOptions (#122)

* fix: use unstable payload option for protojson codec

* refactor: rename is_unstable -> is_non_deterministic

* refactor: use unstable_serialization on the WASM level

* chore: format code

* chore: remove go cache

Author: ciricc

.gitignore

@@ -1,2 +1,3 @@
 test_report
 .restate
+shared-core/pkg/

encoding/encoding.go

@@ -122,6 +122,23 @@ type PayloadCodec interface {
 	Codec
 }
 
+// NonDeterministicSerializer is an interface that codecs can implement to indicate
+// they may produce non-deterministic output for the same input.
+type NonDeterministicSerializer interface {
+	IsNonDeterministic() bool
+}
+
+// IsNonDeterministicSerialization returns true if the codec may produce non-deterministic output.
+// This is true for ProtoJSONCodec (protojson does not guarantee deterministic output)
+// and any codec implementing NonDeterministicSerializer that returns true from IsNonDeterministic().
+func IsNonDeterministicSerialization(codec Codec) bool {
+	if nonDeterministic, ok := codec.(NonDeterministicSerializer); ok {
+		return nonDeterministic.IsNonDeterministic()
+	}
+
+	return false
+}
+
 // InputPayload is provided to Restate upon handler discovery, to teach the ingress how to validate incoming
 // request bodies.
 type InputPayload struct {
@@ -228,6 +245,12 @@ func (p protoCodec) Marshal(output any) (data []byte, err error) {
 
 type protoJSONCodec struct{}
 
+func (j protoJSONCodec) IsNonDeterministic() bool {
+	// protojson does not guarantee deterministic output:
+	// https://github.com/golang/protobuf/issues/1373
+	return true
+}
+
 func (j protoJSONCodec) generateProtoJsonSchema(v any) interface{} {
 	_, msgOk := v.(protoreflect.ProtoMessage)
 	if !msgOk {

encoding/encoding_test.go

@@ -95,3 +95,23 @@ func TestGenerateJsonSchema(t *testing.T) {
 		})
 	}
 }
+
+func TestIsNonDeterministicSerialization(t *testing.T) {
+	tests := []struct {
+		name     string
+		codec    Codec
+		expected bool
+	}{
+		{"JSONCodec", JSONCodec, false},
+		{"ProtoCodec", ProtoCodec, false},
+		{"ProtoJSONCodec", ProtoJSONCodec, true},
+		{"BinaryCodec", BinaryCodec, false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := IsNonDeterministicSerialization(tt.codec)
+			require.Equal(t, tt.expected, result, "IsNonDeterministicSerialization(%s) = %v, want %v", tt.name, result, tt.expected)
+		})
+	}
+}