Merge #152908

152908: roachprod: add codec util r=golgeek a=herkolategan

This change introduces a new codec utility to facilitate serializing and
deserializing `interface{}`.

This is the first step in being able to serialize mixedversion test plans. Since
the mixedversion framework utilizes dynamic types (interfaces{}), for `steps`, we
need a way to be able to handle those types when serializing or deserializing
plans.

See the `README.md` in the `codec` package for a more detailed description of how the
codec package works, and what it is used for.

As part of this PR and as an example we use the new codec to implement serialization
for a list of `ClusterSettingOption`. Cluster settings are used by the mixedversion steps,
which we intend to serialize eventually.

This PR also supplies a `typegen` command that generates registration code for the
concrete types we wish to serialize as part of a dynamic type.

Informs: #149451, #151461

Epic: None
Release note: None

Co-authored-by: Herko Lategan <[email protected]>

Author: craig[bot]

pkg/BUILD.bazel

@@ -327,6 +327,7 @@ ALL_TESTS = [
     "//pkg/roachprod/opentelemetry:opentelemetry_test",
     "//pkg/roachprod/prometheus:prometheus_test",
     "//pkg/roachprod/promhelperclient:promhelperclient_test",
+    "//pkg/roachprod/roachprodutil/codec:codec_test",
     "//pkg/roachprod/ssh:ssh_test",
     "//pkg/roachprod/vm/aws:aws_test",
     "//pkg/roachprod/vm/azure:azure_test",
@@ -1716,6 +1717,10 @@ GO_TARGETS = [
     "//pkg/roachprod/prometheus:prometheus_test",
     "//pkg/roachprod/promhelperclient:promhelperclient",
     "//pkg/roachprod/promhelperclient:promhelperclient_test",
+    "//pkg/roachprod/roachprodutil/codec/cmd/typegen:typegen",
+    "//pkg/roachprod/roachprodutil/codec/cmd/typegen:typegen_lib",
+    "//pkg/roachprod/roachprodutil/codec:codec",
+    "//pkg/roachprod/roachprodutil/codec:codec_test",
     "//pkg/roachprod/roachprodutil:roachprodutil",
     "//pkg/roachprod/ssh:ssh",
     "//pkg/roachprod/ssh:ssh_test",

pkg/gen/misc.bzl

@@ -11,6 +11,7 @@ MISC_SRCS = [
     "//pkg/kv/kvserver/concurrency:keylocks_interval_btree_test.go",
     "//pkg/kv/kvserver/spanlatch:latch_interval_btree.go",
     "//pkg/kv/kvserver/spanlatch:latch_interval_btree_test.go",
+    "//pkg/roachprod/install:types_generated.go",
     "//pkg/roachprod/vm/aws:terraform/main.tf",
     "//pkg/spanconfig/spanconfigstore:entry_interval_btree.go",
     "//pkg/spanconfig/spanconfigstore:entry_interval_btree_test.go",

pkg/roachprod/install/BUILD.bazel

@@ -16,6 +16,7 @@ go_library(
         "services.go",
         "session.go",
         "staging.go",
+        ":typegen-cluster-settings",  # keep
     ],
     embedsrcs = [
         "files/cockroachdb-logging.yaml",
@@ -34,6 +35,7 @@ go_library(
         "//pkg/roachprod/errors",
         "//pkg/roachprod/logger",
         "//pkg/roachprod/roachprodutil",
+        "//pkg/roachprod/roachprodutil/codec",
         "//pkg/roachprod/ssh",
         "//pkg/roachprod/ui",
         "//pkg/roachprod/vm",
@@ -49,6 +51,7 @@ go_library(
         "@com_github_alessio_shellescape//:shellescape",
         "@com_github_cockroachdb_errors//:errors",
         "@com_github_cockroachdb_version//:version",
+        "@in_gopkg_yaml_v3//:yaml_v3",
         "@org_golang_x_exp//maps",
         "@org_golang_x_sync//errgroup",
         "@org_golang_x_sys//unix",
@@ -58,6 +61,7 @@ go_library(
 go_test(
     name = "install_test",
     srcs = [
+        "cluster_settings_test.go",
         "cluster_synced_test.go",
         "cockroach_test.go",
         "expander_test.go",
@@ -84,5 +88,22 @@ go_test(
         "@com_github_cockroachdb_datadriven//:datadriven",
         "@com_github_cockroachdb_errors//:errors",
         "@com_github_stretchr_testify//require",
+        "@in_gopkg_yaml_v3//:yaml_v3",
+    ],
+)
+
+genrule(
+    name = "typegen-cluster-settings",
+    srcs = ["cluster_settings.go"],
+    outs = ["types_generated.go"],
+    cmd = """$(location //pkg/roachprod/roachprodutil/codec/cmd/typegen:typegen) \
+       -output $@ $(location cluster_settings.go)
+""",
+    tools = [
+        "//pkg/roachprod/roachprodutil/codec/cmd/typegen",
+    ],
+    visibility = [
+        ":__pkg__",
+        "//pkg/gen:__pkg__",
     ],
 )

pkg/roachprod/install/cluster_settings.go

@@ -10,8 +10,10 @@ import (
 	"os"
 
 	"github.com/cockroachdb/cockroach/pkg/roachprod/config"
+	"github.com/cockroachdb/cockroach/pkg/roachprod/roachprodutil/codec"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/vm/gce"
 	"github.com/cockroachdb/errors"
+	"gopkg.in/yaml.v3"
 )
 
 // ClusterSettings contains various knobs that affect operations on a cluster.
@@ -41,10 +43,12 @@ type secureFlagsOpt struct {
 
 // ClusterSettingOption is the interface satisfied by options to MakeClusterSettings.
 type ClusterSettingOption interface {
+	codec.DynamicType
 	apply(settings *ClusterSettings)
 }
 
 // ClusterSettingsOption adds cluster settings via SET CLUSTER SETTING.
+// typegen:reg
 type ClusterSettingsOption map[string]string
 
 func (o ClusterSettingsOption) apply(settings *ClusterSettings) {
@@ -54,20 +58,23 @@ func (o ClusterSettingsOption) apply(settings *ClusterSettings) {
 }
 
 // TagOption is used to pass a process tag.
+// typegen:reg
 type TagOption string
 
 func (o TagOption) apply(settings *ClusterSettings) {
 	settings.Tag = string(o)
 }
 
 // BinaryOption is used to pass a process tag.
+// typegen:reg
 type BinaryOption string
 
 func (o BinaryOption) apply(settings *ClusterSettings) {
 	settings.Binary = string(o)
 }
 
 // PGUrlCertsDirOption is used to pass certs dir for secure connections.
+// typegen:reg
 type PGUrlCertsDirOption string
 
 func (o PGUrlCertsDirOption) apply(settings *ClusterSettings) {
@@ -76,6 +83,7 @@ func (o PGUrlCertsDirOption) apply(settings *ClusterSettings) {
 
 // ComplexSecureOption is a complex type for secure options that keeps track of
 // the user's intent regarding security.
+// typegen:reg
 type ComplexSecureOption secureFlagsOpt
 
 func (o ComplexSecureOption) apply(settings *ClusterSettings) {
@@ -127,6 +135,7 @@ func (o ComplexSecureOption) overrideBasedOnClusterSettings(c *SyncedCluster) er
 
 // SimpleSecureOption is a simple type that simplifies setting the secure flags
 // in the cluster settings without keeping track of --secure or --insecure options.
+// typegen:reg
 type SimpleSecureOption bool
 
 func (o SimpleSecureOption) apply(settings *ClusterSettings) {
@@ -151,13 +160,15 @@ type SecureOption interface {
 }
 
 // UseTreeDistOption is passed to use treedist copy algorithm.
+// typegen:reg
 type UseTreeDistOption bool
 
 func (o UseTreeDistOption) apply(settings *ClusterSettings) {
 	settings.UseTreeDist = bool(o)
 }
 
 // EnvOption is used to pass environment variables to the cockroach process.
+// typegen:reg
 type EnvOption []string
 
 var _ EnvOption
@@ -167,6 +178,7 @@ func (o EnvOption) apply(settings *ClusterSettings) {
 }
 
 // NumRacksOption is used to pass the number of racks to partition the nodes into.
+// typegen:reg
 type NumRacksOption int
 
 var _ NumRacksOption
@@ -176,6 +188,7 @@ func (o NumRacksOption) apply(settings *ClusterSettings) {
 }
 
 // DebugDirOption is used to stash debug information.
+// typegen:reg
 type DebugDirOption string
 
 var _ DebugDirOption
@@ -202,3 +215,20 @@ func MakeClusterSettings(opts ...ClusterSettingOption) ClusterSettings {
 	}
 	return clusterSettings
 }
+
+// ClusterSettingOptionList is a list of ClusterSettingOption that can be
+// serialized to YAML. It uses codec.ListWrapper to handle the dynamic types.
+type ClusterSettingOptionList []ClusterSettingOption
+
+func (o ClusterSettingOptionList) MarshalYAML() (any, error) {
+	return codec.WrapList(o), nil
+}
+
+func (o *ClusterSettingOptionList) UnmarshalYAML(value *yaml.Node) error {
+	var lw codec.ListWrapper[ClusterSettingOption]
+	if err := value.Decode(&lw); err != nil {
+		return err
+	}
+	*o = lw.Get()
+	return nil
+}

pkg/roachprod/install/cluster_settings_test.go

@@ -0,0 +1,32 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+package install
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+func TestClusterSettingOptionListCodec(t *testing.T) {
+	opts := ClusterSettingOptionList{
+		NumRacksOption(1),
+		DebugDirOption("foo"),
+	}
+	data, err := yaml.Marshal(opts)
+	require.NoError(t, err)
+
+	var decOpts ClusterSettingOptionList
+	dec := yaml.NewDecoder(bytes.NewReader(data))
+	dec.KnownFields(true)
+	err = dec.Decode(&decOpts)
+	require.NoError(t, err)
+
+	require.Equal(t, opts, decOpts)
+	require.Equal(t, MakeClusterSettings(opts...), MakeClusterSettings(decOpts...))
+}

pkg/roachprod/install/types_generated.go

@@ -0,0 +1,22 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "codec",
+    srcs = ["types.go"],
+    importpath = "github.com/cockroachdb/cockroach/pkg/roachprod/roachprodutil/codec",
+    visibility = ["//visibility:public"],
+    deps = ["@in_gopkg_yaml_v3//:yaml_v3"],
+)
+
+go_test(
+    name = "codec_test",
+    srcs = [
+        "types_registry_test.go",
+        "types_test.go",
+    ],
+    embed = [":codec"],
+    deps = [
+        "@com_github_stretchr_testify//require",
+        "@in_gopkg_yaml_v3//:yaml_v3",
+    ],
+)