Skip to content

Commit 95b3cab

Browse files
committed
roachprod: add codec util
This change introduces a new codec utility to facilitate serializing and deserializing `interface{}`. This is the first step in being able to serialize mixedversion test plans. Since the mixedversion framework utilizes dynamic types (interfaces{}), for `steps`, we need a way to be able to handle those types when serializing or deserializing plans. Informs: #149451, #151461 Epic: None Release note: None
1 parent 8197306 commit 95b3cab

File tree

6 files changed

+433
-0
lines changed

6 files changed

+433
-0
lines changed

pkg/BUILD.bazel

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,7 @@ ALL_TESTS = [
324324
"//pkg/roachprod/opentelemetry:opentelemetry_test",
325325
"//pkg/roachprod/prometheus:prometheus_test",
326326
"//pkg/roachprod/promhelperclient:promhelperclient_test",
327+
"//pkg/roachprod/roachprodutil/codec:codec_test",
327328
"//pkg/roachprod/ssh:ssh_test",
328329
"//pkg/roachprod/vm/aws:aws_test",
329330
"//pkg/roachprod/vm/azure:azure_test",
@@ -1698,6 +1699,8 @@ GO_TARGETS = [
16981699
"//pkg/roachprod/prometheus:prometheus_test",
16991700
"//pkg/roachprod/promhelperclient:promhelperclient",
17001701
"//pkg/roachprod/promhelperclient:promhelperclient_test",
1702+
"//pkg/roachprod/roachprodutil/codec:codec",
1703+
"//pkg/roachprod/roachprodutil/codec:codec_test",
17011704
"//pkg/roachprod/roachprodutil:roachprodutil",
17021705
"//pkg/roachprod/ssh:ssh",
17031706
"//pkg/roachprod/ssh:ssh_test",
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
2+
3+
go_library(
4+
name = "codec",
5+
srcs = ["types.go"],
6+
importpath = "github.com/cockroachdb/cockroach/pkg/roachprod/roachprodutil/codec",
7+
visibility = ["//visibility:public"],
8+
deps = ["@in_gopkg_yaml_v3//:yaml_v3"],
9+
)
10+
11+
go_test(
12+
name = "codec_test",
13+
srcs = [
14+
"types_registry_test.go",
15+
"types_test.go",
16+
],
17+
embed = [":codec"],
18+
deps = [
19+
"@com_github_stretchr_testify//require",
20+
"@in_gopkg_yaml_v3//:yaml_v3",
21+
],
22+
)
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
# Introduction
2+
3+
The codec package provides a way of serializing and deserializing data structures, with YAML,
4+
that contain interfaces (referred to as dynamic types in this package)
5+
6+
For example:
7+
``` go
8+
struct Foo {
9+
Bar interface{}
10+
}
11+
```
12+
13+
Typically, this would serialize without incident, using the standard YAML
14+
library, and whichever concrete data `Bar` holds will be serialized.
15+
16+
17+
However, challenges emerge during deserialization back to the original
18+
structure. Without type information, the decoder cannot determine what concrete
19+
type `Bar` originally held, resulting in deserialization to generic containers
20+
like `map[string]interface{}`.
21+
22+
Ideally, we want to preserve the original type information, enabling us to
23+
maintain the polymorphic behavior associated with the original interface
24+
implementation.
25+
26+
# Usage
27+
28+
Consider the following example types:
29+
``` go
30+
type (
31+
Animal interface {
32+
codec.DynamicType // Requierd for dynamic type serialization
33+
Speak() string
34+
}
35+
36+
Dog struct {
37+
Name string
38+
}
39+
40+
Cat struct {
41+
Name string
42+
}
43+
)
44+
45+
func (d Dog) Speak() string {
46+
return d.Name + " woofs"
47+
}
48+
49+
func (c Cat) Speak() string {
50+
return c.Name + " meows"
51+
}
52+
```
53+
54+
The `codec.DynamicType` should be added to any type that is to be serialized
55+
as a dynamic type.
56+
57+
## Registration
58+
59+
In order for the codec package decoder to recognize the types, they must be
60+
registered and implement the `codec.DynamicType`. All concrete types that
61+
implement `codec.DynamicType` must be registered.
62+
63+
A custom `TypeName` can be provided for each type, but it's preferred to use the
64+
`codec.ResolveTypeName` function to generate a unique name for each type.
65+
66+
``` go
67+
func init() {
68+
codec.Register(new(Dog))
69+
codec.Register(new(Cat))
70+
}
71+
72+
func (d Dog) GetTypeName() codec.TypeName {
73+
return codec.ResolveTypeName(d)
74+
}
75+
76+
func (c Cat) GetTypeName() codec.TypeName {
77+
return codec.ResolveTypeName(c)
78+
}
79+
```
80+
81+
## Encoding and decoding
82+
83+
Example of a structure ready to be serialized:
84+
``` go
85+
type struct Data {
86+
animals codec.ListWrapper[Animal]
87+
favouriteAnimal codec.Wrapper[Animal]
88+
extinctAnimal codec.Wrapper[*Animal] // Pointers can be used as well
89+
}
90+
```
91+
92+
The structure can then be serialized and deserialized as usual:
93+
``` go
94+
var data Data
95+
// ...
96+
buf, err := yaml.Marshal(data)
97+
// ...
98+
dec := yaml.NewDecoder(bytes.NewReader(buf))
99+
dec.KnownFields(true)
100+
err = dec.Decode(&data)
101+
```
102+
103+
### Convenience methods
104+
105+
The wrappers provide a convenient `Get` method that retrieves the underlying
106+
value with proper type information preserved, leveraging Go's generics system to
107+
ensure type safety without manual type assertions.
108+
109+
``` go
110+
var animal Animal = data.favouriteAnimal.Get()
111+
var animals []Animal = data.animals.Get()
112+
data.extinctAnimal.Get().Speak()
113+
```
Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
// Copyright 2025 The Cockroach Authors.
2+
//
3+
// Use of this software is governed by the CockroachDB Software License
4+
// included in the /LICENSE file.
5+
6+
package codec
7+
8+
import (
9+
"fmt"
10+
"path"
11+
"reflect"
12+
"strings"
13+
14+
"gopkg.in/yaml.v3"
15+
)
16+
17+
type (
18+
// TypeMap is a map of type names with their corresponding `reflect.Type`.
19+
// This is used to resolve type names to `reflect.Type` during decoding.
20+
TypeMap map[TypeName]reflect.Type
21+
22+
// Metadata is an internal type used to encode and decode dynamic types with
23+
// their type name.
24+
metadata struct {
25+
FieldType fieldType `yaml:"type"`
26+
Val DynamicType `yaml:"val"`
27+
}
28+
29+
// DynamicType is an interface used to encode and decode dynamic types.
30+
// This is used to allow for custom types to be encoded and decoded.
31+
DynamicType interface {
32+
GetTypeName() TypeName
33+
}
34+
35+
// Wrapper wraps a dynamic type for encoding and decoding.
36+
Wrapper[T DynamicType] struct {
37+
Val T `yaml:",inline"`
38+
}
39+
40+
// ListWrapper wraps a list of dynamic types for encoding and decoding.
41+
ListWrapper[T DynamicType] struct {
42+
Val []T `yaml:",inline"`
43+
}
44+
45+
// TypeName stores the package and name of a type.
46+
TypeName struct {
47+
pkg string
48+
name string
49+
}
50+
51+
// fieldType provides the type information during encoding and decoding. It
52+
// stores a string representation of the TypeName and whether the type should
53+
// be decoded as a pointer.
54+
// Ex. `util.Tree` or `util.*Tree`
55+
fieldType string
56+
)
57+
58+
var typeRegistry = make(TypeMap)
59+
60+
// Register registers a dynamic type with the codec package. All dynamic types
61+
// must be registered before they can be encoded and decoded.
62+
func Register(t DynamicType) {
63+
typeRegistry[t.GetTypeName()] = ResolveElem(reflect.TypeOf(t))
64+
}
65+
66+
// Wrap is a convenience function for wrapping a dynamic type.
67+
func Wrap[T DynamicType](t T) Wrapper[T] {
68+
return Wrapper[T]{Val: t}
69+
}
70+
71+
// WrapList is a convenience function for wrapping a list of dynamic types.
72+
func WrapList[T DynamicType](t []T) ListWrapper[T] {
73+
return ListWrapper[T]{Val: t}
74+
}
75+
76+
func (w *Wrapper[T]) Get() T {
77+
return w.Val
78+
}
79+
80+
func (w *ListWrapper[T]) Get() []T {
81+
return w.Val
82+
}
83+
84+
// ResolveElem unwraps a pointer type to get the underlying type or returns the
85+
// type itself if it is not a pointer.
86+
func ResolveElem(t reflect.Type) reflect.Type {
87+
if t.Kind() == reflect.Ptr {
88+
return t.Elem()
89+
}
90+
return t
91+
}
92+
93+
// ResolveTypeName resolves the type name of a dynamic type.
94+
func ResolveTypeName(t DynamicType) TypeName {
95+
tType := ResolveElem(reflect.TypeOf(t))
96+
return TypeName{
97+
pkg: path.Base(tType.PkgPath()),
98+
name: tType.Name(),
99+
}
100+
}
101+
102+
// toFieldType encodes the type name as a fieldType (string), including the
103+
// package path, and if the type should later on be decoded as a pointer.
104+
func (t TypeName) toFieldType(pointer bool) fieldType {
105+
name := t.name
106+
if pointer {
107+
name = "*" + name
108+
}
109+
return fieldType(t.pkg + "." + name)
110+
}
111+
112+
// name returns the package and name of the type.
113+
func (t fieldType) name() (TypeName, error) {
114+
s := strings.Split(string(t), ".")
115+
if len(s) != 2 {
116+
return TypeName{}, fmt.Errorf("invalid type name %q", t)
117+
}
118+
// The `typeName` should not include the pointer prefix, as this is only used
119+
// during decoding to determine if the value should be decoded as a pointer.
120+
s[1] = strings.TrimPrefix(s[1], "*")
121+
return TypeName{
122+
pkg: s[0],
123+
name: s[1],
124+
}, nil
125+
}
126+
127+
// isPointer returns true if the type is a pointer.
128+
func (t fieldType) isPointer() bool {
129+
return strings.Contains(string(t), "*")
130+
}
131+
132+
// MarshalYAML implements the [yaml.Marshaler] interface. The wrapper is
133+
// inspected to get its type information, which is then encoded with the
134+
// metadata type.
135+
func (w Wrapper[T]) MarshalYAML() (any, error) {
136+
typeName := w.Get().GetTypeName()
137+
dynType := reflect.TypeOf(w.Get())
138+
pointer := false
139+
if dynType.Kind() == reflect.Ptr {
140+
pointer = true
141+
}
142+
return metadata{
143+
FieldType: typeName.toFieldType(pointer),
144+
Val: w.Val,
145+
}, nil
146+
}
147+
148+
// UnmarshalYAML implements the [yaml.Unmarshaler] interface. Since the wrapper
149+
// was stored with metadata, we decode the type information and then decode the
150+
// value using the type information.
151+
func (w *Wrapper[T]) UnmarshalYAML(value *yaml.Node) error {
152+
ft := fieldType(value.Content[1].Value)
153+
typeName, err := ft.name()
154+
if err != nil {
155+
return err
156+
}
157+
objType, ok := typeRegistry[typeName]
158+
if !ok {
159+
return fmt.Errorf("unknown type %s.%s", typeName.pkg, typeName.name)
160+
}
161+
objValuePtr := reflect.New(objType).Interface()
162+
if err := value.Content[3].Decode(objValuePtr); err != nil {
163+
return err
164+
}
165+
objValue := reflect.ValueOf(objValuePtr).Elem().Interface()
166+
if ft.isPointer() {
167+
objValue = reflect.ValueOf(objValuePtr).Interface()
168+
}
169+
*w = Wrapper[T]{
170+
objValue.(T),
171+
}
172+
return nil
173+
}
174+
175+
// MarshalYAML implements the [yaml.Marshaler] interface. This is a convenience
176+
// function for encoding a list of dynamic types and leverages the Wrapper type
177+
// to encode each dynamic type.
178+
func (w ListWrapper[T]) MarshalYAML() (any, error) {
179+
wrappers := make([]Wrapper[T], len(w.Val))
180+
for i, v := range w.Val {
181+
wrappers[i] = Wrap(v)
182+
}
183+
return wrappers, nil
184+
}
185+
186+
// UnmarshalYAML implements the [yaml.Unmarshaler] interface. This is a
187+
// convenience function for decoding a list of dynamic types and leverages the
188+
// Wrapper type to encode each dynamic type.
189+
func (w *ListWrapper[T]) UnmarshalYAML(value *yaml.Node) error {
190+
*w = ListWrapper[T]{
191+
Val: make([]T, len(value.Content)),
192+
}
193+
for i, v := range value.Content {
194+
var e Wrapper[T]
195+
if err := v.Decode(&e); err != nil {
196+
return err
197+
}
198+
w.Val[i] = e.Val
199+
}
200+
return nil
201+
}
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
// Copyright 2025 The Cockroach Authors.
2+
//
3+
// Use of this software is governed by the CockroachDB Software License
4+
// included in the /LICENSE file.
5+
6+
package codec
7+
8+
func init() {
9+
Register(new(Dog))
10+
Register(new(Cat))
11+
}
12+
13+
func (d Dog) GetTypeName() TypeName {
14+
return ResolveTypeName(d)
15+
}
16+
17+
func (c Cat) GetTypeName() TypeName {
18+
return ResolveTypeName(c)
19+
}

0 commit comments

Comments
 (0)