Customizable serialization (#8426)

## What changed?

Added `TEMPORAL_TEST_DATA_ENCODING` to change DataBlob encoding from
"proto3" to "json".

## Why?

Observability and debugging. The ability to see payloads decoded in
debugger and OTEL traces is valuable.

## How did you test it?
- [ ] built
- [ ] run locally and tested manually
- [ ] covered by existing tests
- [ ] added new unit test(s)
- [x] added new functional test(s)

## Potential risks

End users should never use this. The env var name therefore includes
`TEST_`.

Author: stephanos

common/dynamicconfig/collection_test.go

@@ -38,6 +38,7 @@ const (
 	testGetDurationPropertyStructuredDefaults         = "testGetDurationPropertyStructuredDefaults"
 	testGetBoolPropertyFilteredByNamespaceIDKey       = "testGetBoolPropertyFilteredByNamespaceIDKey"
 	testGetBoolPropertyFilteredByTaskQueueInfoKey     = "testGetBoolPropertyFilteredByTaskQueueInfoKey"
+	testGetStringPropertyFilteredByNamespaceKey       = "testGetStringPropertyFilteredByNamespaceKey"
 	testGetStringPropertyFilteredByNamespaceIDKey     = "testGetStringPropertyFilteredByNamespaceIDKey"
 	testGetIntPropertyFilteredByDestinationKey        = "testGetIntPropertyFilteredByDestinationKey"
 )
@@ -87,17 +88,17 @@ func (s *collectionSuite) TestGetIntPropertyFilteredByNamespace() {
 }
 
 func (s *collectionSuite) TestGetStringPropertyFnFilteredByNamespace() {
-	namespace := "testNamespace"
-	value := dynamicconfig.DefaultEventEncoding.Get(s.cln)
-	// copied default value, change this if it changes
-	s.Equal(enumspb.ENCODING_TYPE_PROTO3.String(), value(namespace))
-	s.client.SetValue(dynamicconfig.DefaultEventEncoding.Key().String(), "efg")
-	s.Equal("efg", value(namespace))
+	ns := "testNamespace"
+	setting := dynamicconfig.NewNamespaceStringSetting(testGetStringPropertyFilteredByNamespaceKey, "abc", "")
+	value := setting.Get(s.cln)
+	s.Equal("abc", value(ns))
+	s.client.SetValue(testGetStringPropertyFilteredByNamespaceKey, "efg")
+	s.Equal("efg", value(ns))
 }
 
 func (s *collectionSuite) TestGetStringPropertyFnFilteredByNamespaceID() {
-	setting := dynamicconfig.NewNamespaceIDStringSetting(testGetStringPropertyFilteredByNamespaceIDKey, "abc", "")
 	namespaceID := namespace.ID("testNamespaceID")
+	setting := dynamicconfig.NewNamespaceIDStringSetting(testGetStringPropertyFilteredByNamespaceIDKey, "abc", "")
 	value := setting.Get(s.cln)
 	s.Equal("abc", value(namespaceID))
 	s.client.SetValue(testGetStringPropertyFilteredByNamespaceIDKey, "efg")

common/dynamicconfig/constants.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"time"
 
-	enumspb "go.temporal.io/api/enums/v1"
 	sdkworker "go.temporal.io/sdk/worker"
 	"go.temporal.io/server/common/debug"
 	"go.temporal.io/server/common/primitives"
@@ -2307,11 +2306,6 @@ When the this config is zero or lower we will only update shard info at most onc
 		false,
 		`EmitShardLagLog whether emit the shard lag log`,
 	)
-	DefaultEventEncoding = NewNamespaceStringSetting(
-		"history.defaultEventEncoding",
-		enumspb.ENCODING_TYPE_PROTO3.String(),
-		`DefaultEventEncoding is the encoding type for history events`,
-	)
 	DefaultActivityRetryPolicy = NewNamespaceTypedSetting(
 		"history.defaultActivityRetryPolicy",
 		retrypolicy.DefaultDefaultRetrySettings,

common/persistence/serialization/codec.go

@@ -2,13 +2,42 @@ package serialization
 
 import (
 	"errors"
+	"fmt"
+	"os"
+	"strings"
 
 	commonpb "go.temporal.io/api/common/v1"
 	enumspb "go.temporal.io/api/enums/v1"
 	"go.temporal.io/server/common/codec"
 	"google.golang.org/protobuf/proto"
 )
 
+// SerializerDataEncodingEnvVar controls which codec is used for encoding DataBlobs.
+//
+// Currently supported values (case-insensitive):
+//   - "json"
+//   - "proto3"
+//
+// Decoding always support all encodings regardless of this setting.
+//
+// WARNING: This environment variable should only be used for testing; and never set it in production.
+const SerializerDataEncodingEnvVar = "TEMPORAL_TEST_DATA_ENCODING"
+
+// EncodingTypeFromEnv returns an EncodingType based on the environment variable `TEMPORAL_TEST_DATA_ENCODING`.
+// It defaults to "ENCODING_TYPE_PROTO3" codec if the environment variable is not set.
+func EncodingTypeFromEnv() enumspb.EncodingType {
+	codecType := os.Getenv(SerializerDataEncodingEnvVar)
+	switch strings.ToLower(codecType) {
+	case "", "proto3":
+		return enumspb.ENCODING_TYPE_PROTO3
+	case "json":
+		return enumspb.ENCODING_TYPE_JSON
+	default:
+		//nolint:forbidigo // should fail fast and hard if used incorrectly
+		panic(fmt.Sprintf("unknown codec %q for environment variable %s", codecType, SerializerDataEncodingEnvVar))
+	}
+}
+
 // ProtoEncode is kept for backward compatibility.
 func ProtoEncode(m proto.Message) (*commonpb.DataBlob, error) {
 	return encodeBlob(m, enumspb.ENCODING_TYPE_PROTO3)

common/persistence/serialization/serializer.go

@@ -18,14 +18,20 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
-// DefaultDecoder is here for convenience to skip the need to create a new Serializer when only decodig is needed.
-// It does not need an encoding type; as it will use the one defined in the DataBlob.
-var r Serializer = &serializerImpl{encodingType: enumspb.ENCODING_TYPE_UNSPECIFIED}
-var DefaultDecoder Decoder = r
+var (
+	// DefaultDecoder is here for convenience to skip the need to create a new Serializer when only decoding is needed.
+	// It does not need an encoding type; as it will use the one defined in the DataBlob.
+	defaultSerializer Serializer = &serializerImpl{encodingType: enumspb.ENCODING_TYPE_UNSPECIFIED}
+	DefaultDecoder    Decoder    = defaultSerializer
+
+	// proto3Encoder always encodes as proto3, used by EnsureProto3Encoding.
+	proto3Encoder Encoder = &serializerImpl{encodingType: enumspb.ENCODING_TYPE_PROTO3}
+)
 
 type (
 	// Encoder is used to encode objects to DataBlobs.
 	Encoder interface {
+		EncodingType() enumspb.EncodingType
 		SerializeEvents(batch []*historypb.HistoryEvent) (*commonpb.DataBlob, error)
 		SerializeEvent(event *historypb.HistoryEvent) (*commonpb.DataBlob, error)
 		SerializeClusterMetadata(icm *persistencespb.ClusterMetadata) (*commonpb.DataBlob, error)
@@ -133,7 +139,11 @@ type (
 )
 
 func NewSerializer() Serializer {
-	return &serializerImpl{encodingType: enumspb.ENCODING_TYPE_PROTO3}
+	return &serializerImpl{encodingType: EncodingTypeFromEnv()}
+}
+
+func (t *serializerImpl) EncodingType() enumspb.EncodingType {
+	return t.encodingType
 }
 
 func (t *serializerImpl) SerializeTask(
@@ -666,3 +676,25 @@ func (t *serializerImpl) QueueStateFromBlob(data *commonpb.DataBlob) (*persisten
 	result := &persistencespb.QueueState{}
 	return result, Decode(data, result)
 }
+
+// ReencodeEventBlobsAsProto3 re-encodes event blobs as proto3 if the serializer uses a different encoding.
+// In production (proto3 encoding), this returns the input unchanged.
+func ReencodeEventBlobsAsProto3(serializer Serializer, blobs []*commonpb.DataBlob) ([]*commonpb.DataBlob, error) {
+	if serializer.EncodingType() == enumspb.ENCODING_TYPE_PROTO3 || len(blobs) == 0 {
+		return blobs, nil
+	}
+
+	// Re-encode all blobs as proto3.
+	result := make([]*commonpb.DataBlob, len(blobs))
+	for i, blob := range blobs {
+		events, err := serializer.DeserializeEvents(blob)
+		if err != nil {
+			return nil, err
+		}
+		result[i], err = proto3Encoder.SerializeEvents(events)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return result, nil
+}

docs/development/testing.md

@@ -16,6 +16,7 @@ This document describes the project's testing setup, utilities and best practice
 - `TEMPORAL_TEST_OTEL_OUTPUT`: Enables OpenTelemetry (OTEL) trace output for failed tests to the provided file path.
 - `TEMPORAL_TEST_SHARED_CLUSTERS`: Number of shared clusters in the pool. Each can be used by multiple tests simultaneously.
 - `TEMPORAL_TEST_DEDICATED_CLUSTERS`: Number of dedicated clusters in the pool. Each can be used by one test only at a time.
+- `TEMPORAL_TEST_DATA_ENCODING`: Controls the encoding used for persistence DataBlobs. Available options: `proto3` (default) or `json`.
 
 ### Debugging via IDE
 

service/history/api/get_history_util.go

@@ -135,6 +135,14 @@ func GetRawHistory(
 			rawHistory = append(rawHistory, blob)
 		}
 	}
+
+	// Ensure all raw history is proto3 encoded since data may be stored in other formats during testing.
+	// In production (proto3 encoding), this returns the input unchanged.
+	rawHistory, err = serialization.ReencodeEventBlobsAsProto3(shardContext.GetPayloadSerializer(), rawHistory)
+	if err != nil {
+		return nil, nil, err
+	}
+
 	return rawHistory, nextToken, nil
 }
 

service/history/api/getworkflowexecutionrawhistory/api.go

@@ -13,6 +13,7 @@ import (
 	"go.temporal.io/server/common/metrics"
 	"go.temporal.io/server/common/namespace"
 	"go.temporal.io/server/common/persistence"
+	"go.temporal.io/server/common/persistence/serialization"
 	"go.temporal.io/server/common/persistence/versionhistory"
 	"go.temporal.io/server/common/rpc/interceptor"
 	"go.temporal.io/server/service/history/api"
@@ -130,9 +131,16 @@ func Invoke(
 		metrics.OperationTag(metrics.AdminGetWorkflowExecutionRawHistoryScope),
 	)
 
+	// Ensure all raw history is proto3 encoded since data may be stored in other formats during testing.
+	// In production (proto3 encoding), this returns the input unchanged.
+	historyBlobs, err := serialization.ReencodeEventBlobsAsProto3(shardContext.GetPayloadSerializer(), rawHistoryResponse.HistoryEventBlobs)
+	if err != nil {
+		return nil, err
+	}
+
 	result :=
 		&adminservice.GetWorkflowExecutionRawHistoryResponse{
-			HistoryBatches: rawHistoryResponse.HistoryEventBlobs,
+			HistoryBatches: historyBlobs,
 			VersionHistory: targetVersionHistory,
 			HistoryNodeIds: rawHistoryResponse.NodeIDs,
 		}

service/history/api/getworkflowexecutionrawhistoryv2/api.go

@@ -13,6 +13,7 @@ import (
 	"go.temporal.io/server/common/metrics"
 	"go.temporal.io/server/common/namespace"
 	"go.temporal.io/server/common/persistence"
+	"go.temporal.io/server/common/persistence/serialization"
 	"go.temporal.io/server/common/persistence/versionhistory"
 	"go.temporal.io/server/common/rpc/interceptor"
 	"go.temporal.io/server/service/history/api"
@@ -136,9 +137,16 @@ func Invoke(
 		metrics.OperationTag(metrics.AdminGetWorkflowExecutionRawHistoryV2Scope),
 	)
 
+	// Ensure all raw history is proto3 encoded since data may be stored in other formats during testing.
+	// In production (proto3 encoding), this returns the input unchanged.
+	historyBlobs, err := serialization.ReencodeEventBlobsAsProto3(shardContext.GetPayloadSerializer(), rawHistoryResponse.HistoryEventBlobs)
+	if err != nil {
+		return nil, err
+	}
+
 	result :=
 		&adminservice.GetWorkflowExecutionRawHistoryV2Response{
-			HistoryBatches: rawHistoryResponse.HistoryEventBlobs,
+			HistoryBatches: historyBlobs,
 			VersionHistory: targetVersionHistory,
 			HistoryNodeIds: rawHistoryResponse.NodeIDs,
 		}

service/history/configs/config.go

@@ -200,8 +200,6 @@ type Config struct {
 	// right now only used by GetMutableState
 	LongPollExpirationInterval dynamicconfig.DurationPropertyFnWithNamespaceFilter
 
-	// encoding the history events
-	EventEncodingType dynamicconfig.StringPropertyFnWithNamespaceFilter
 	// whether or not using ParentClosePolicy
 	EnableParentClosePolicy dynamicconfig.BoolPropertyFnWithNamespaceFilter
 	// whether or not enable system workers for processing parent close policy task
@@ -617,7 +615,6 @@ func NewConfig(
 		// history client: client/history/client.go set the client timeout 30s
 		// TODO: Return this value to the client: go.temporal.io/server/issues/294
 		LongPollExpirationInterval:          dynamicconfig.HistoryLongPollExpirationInterval.Get(dc),
-		EventEncodingType:                   dynamicconfig.DefaultEventEncoding.Get(dc),
 		EnableParentClosePolicy:             dynamicconfig.EnableParentClosePolicy.Get(dc),
 		NumParentClosePolicySystemWorkflows: dynamicconfig.NumParentClosePolicySystemWorkflows.Get(dc),
 		EnableParentClosePolicyWorker:       dynamicconfig.EnableParentClosePolicyWorker.Get(dc),