diff --git a/go.mod b/go.mod index fd16db69c6..000a7f02b5 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/jmoiron/sqlx v1.4.0 github.com/jonboulle/clockwork v0.4.0 github.com/jpillora/backoff v1.0.0 + github.com/kylelemons/godebug v1.1.0 github.com/lib/pq v1.10.9 github.com/marcboeker/go-duckdb v1.8.3 github.com/pelletier/go-toml/v2 v2.2.3 @@ -53,7 +54,6 @@ require ( go.opentelemetry.io/otel/sdk/log v0.6.0 go.opentelemetry.io/otel/sdk/metric v1.35.0 go.opentelemetry.io/otel/trace v1.35.0 - go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.36.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 @@ -121,6 +121,7 @@ require ( go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect go.opentelemetry.io/proto/otlp v1.5.0 // indirect + go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.24.0 // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/sync v0.12.0 // indirect diff --git a/pkg/config/configdoc/configdoc.go b/pkg/config/configdoc/configdoc.go new file mode 100644 index 0000000000..95714193bb --- /dev/null +++ b/pkg/config/configdoc/configdoc.go @@ -0,0 +1,217 @@ +package configdoc + +import ( + "errors" + "fmt" + "strings" + + "github.com/smartcontractkit/chainlink-common/pkg/config" +) + +const ( + FieldDefault = "# Default" + FieldExample = "# Example" + + TokenAdvanced = "**ADVANCED**" +) + +// Generate returns MarkDown documentation generated from the TOML string. +// - Each field but include a trailing comment of either FieldDefault or FieldExample. +// - If a description begins with TokenAdvanced, then a warning will be included. +// - The markdown wil begin with the header, followed by the example +// - Extended descriptions can be applied to top level tables +func Generate(toml, header, example string, extendedDescriptions map[string]string) (string, error) { + items, err := parseTOMLDocs(toml, extendedDescriptions) + var sb strings.Builder + + sb.WriteString(header) + sb.WriteString(` +## Example + +`) + sb.WriteString("```toml\n") + sb.WriteString(example) + sb.WriteString("\n```\n\n") + + for _, item := range items { + sb.WriteString(item.String()) + sb.WriteString("\n\n") + } + + return sb.String(), err +} + +func advancedWarning(msg string) string { + return fmt.Sprintf(":warning: **_ADVANCED_**: _%s_\n", msg) +} + +// lines holds a set of contiguous lines +type lines []string + +func (d lines) String() string { + return strings.Join(d, "\n") +} + +type table struct { + name string + codes lines + adv bool + desc lines + extended string +} + +func newTable(line string, desc lines, extendedDescriptions map[string]string) *table { + t := &table{ + name: strings.Trim(line, "[]"), + codes: []string{line}, + desc: desc, + } + if extended, ok := extendedDescriptions[t.name]; ok { + t.extended = extended + } + if len(desc) > 0 { + if strings.HasPrefix(strings.TrimSpace(desc[0]), TokenAdvanced) { + t.adv = true + t.desc = t.desc[1:] + } + } + return t +} + +func newArrayOfTables(line string, desc lines, extendedDescriptions map[string]string) *table { + t := &table{ + name: strings.Trim(strings.Trim(line, FieldExample), "[]"), + codes: []string{line}, + desc: desc, + } + if extended, ok := extendedDescriptions[t.name]; ok { + t.extended = extended + } + if len(desc) > 0 { + if strings.HasPrefix(strings.TrimSpace(desc[0]), TokenAdvanced) { + t.adv = true + t.desc = t.desc[1:] + } + } + return t +} + +func (t table) advanced() string { + if t.adv { + return advancedWarning("Do not change these settings unless you know what you are doing.") + } + return "" +} + +func (t table) code() string { + if t.extended == "" { + return fmt.Sprint("```toml\n", t.codes, "\n```\n") + } + return "" +} + +// String prints a table as an H2, followed by a code block and description. +func (t *table) String() string { + return fmt.Sprint("## ", t.name, "\n", + t.advanced(), + t.code(), + t.desc, + t.extended) +} + +type keyval struct { + name string + code string + adv bool + desc lines +} + +func newKeyval(line string, desc lines) keyval { + line = strings.TrimSpace(line) + kv := keyval{ + name: line[:strings.Index(line, " ")], + code: line, + desc: desc, + } + if len(desc) > 0 && strings.HasPrefix(strings.TrimSpace(desc[0]), TokenAdvanced) { + kv.adv = true + kv.desc = kv.desc[1:] + } + return kv +} + +func (k keyval) advanced() string { + if k.adv { + return advancedWarning("Do not change this setting unless you know what you are doing.") + } + return "" +} + +// String prints a keyval as an H3, followed by a code block and description. +func (k keyval) String() string { + name := k.name + if i := strings.LastIndex(name, "."); i > -1 { + name = name[i+1:] + } + return fmt.Sprint("### ", name, "\n", + k.advanced(), + "```toml\n", + k.code, + "\n```\n", + k.desc) +} + +func parseTOMLDocs(s string, extendedDescriptions map[string]string) (items []fmt.Stringer, err error) { + defer func() { _, err = config.MultiErrorList(err) }() + globalTable := table{name: "Global"} + currentTable := &globalTable + items = append(items, currentTable) + var desc lines + for _, line := range strings.Split(s, "\n") { + if strings.HasPrefix(line, "#") { + // comment + desc = append(desc, strings.TrimSpace(line[1:])) + } else if strings.TrimSpace(line) == "" { + // empty + if len(desc) > 0 { + items = append(items, desc) + desc = nil + } + } else if strings.HasPrefix(line, "[[") { + currentTable = newArrayOfTables(line, desc, extendedDescriptions) + items = append(items, currentTable) + desc = nil + } else if strings.HasPrefix(line, "[") { + currentTable = newTable(line, desc, extendedDescriptions) + items = append(items, currentTable) + desc = nil + } else { + kv := newKeyval(line, desc) + shortName := kv.name + if currentTable != &globalTable { + // update to full name + kv.name = currentTable.name + "." + kv.name + } + if len(kv.desc) == 0 { + err = errors.Join(err, fmt.Errorf("%s: missing description", kv.name)) + } else if !strings.HasPrefix(kv.desc[0], shortName) { + err = errors.Join(err, fmt.Errorf("%s: description does not begin with %q", kv.name, shortName)) + } + if !strings.HasSuffix(line, FieldDefault) && !strings.HasSuffix(line, FieldExample) { + err = errors.Join(err, fmt.Errorf(`%s: is not one of %v`, kv.name, []string{FieldDefault, FieldExample})) + } + + items = append(items, kv) + currentTable.codes = append(currentTable.codes, kv.code) + desc = nil + } + } + if len(globalTable.codes) == 0 { + // drop it + items = items[1:] + } + if len(desc) > 0 { + items = append(items, desc) + } + return +} diff --git a/pkg/config/configdoc/configdoc_test.go b/pkg/config/configdoc/configdoc_test.go new file mode 100644 index 0000000000..39703f2d46 --- /dev/null +++ b/pkg/config/configdoc/configdoc_test.go @@ -0,0 +1,37 @@ +package configdoc + +import ( + _ "embed" + "testing" + + "github.com/stretchr/testify/require" +) + +//go:embed testdata/gen_exp.md +var exp string + +func TestGenerate(t *testing.T) { + def := ` +# Foo is a boolean field. +Foo = false # Default +# Bar is a number. +Bar = 42 # Example +[Baz] +# Test holds a string. +Test = "test" # Example` + + header := `# Example docs +This is the header. It has a list: +- first +- second` + + example := `Bar = 10 +Baz.Test = "asdf"` + + s, err := Generate(def, header, example, map[string]string{ + "Baz": "Baz has an extended description", + }) + require.NoError(t, err) + + require.Equal(t, exp, s) +} diff --git a/pkg/config/configdoc/testdata/gen_exp.md b/pkg/config/configdoc/testdata/gen_exp.md new file mode 100644 index 0000000000..33ca006ac4 --- /dev/null +++ b/pkg/config/configdoc/testdata/gen_exp.md @@ -0,0 +1,39 @@ +# Example docs +This is the header. It has a list: +- first +- second +## Example + +```toml +Bar = 10 +Baz.Test = "asdf" +``` + +## Global +```toml +Foo = false # Default +Bar = 42 # Example +``` + + +### Foo +```toml +Foo = false # Default +``` +Foo is a boolean field. + +### Bar +```toml +Bar = 42 # Example +``` +Bar is a number. + +## Baz +Baz has an extended description + +### Test +```toml +Test = "test" # Example +``` +Test holds a string. + diff --git a/pkg/config/configtest/configtest.go b/pkg/config/configtest/configtest.go new file mode 100644 index 0000000000..f0e371de4c --- /dev/null +++ b/pkg/config/configtest/configtest.go @@ -0,0 +1,139 @@ +package configtest + +import ( + "encoding" + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/kylelemons/godebug/diff" + "github.com/pelletier/go-toml/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-common/pkg/config" +) + +// AssertFieldsNotNil recursively checks s for nil fields. s must be a struct. +func AssertFieldsNotNil(t *testing.T, s interface{}) { + t.Helper() + err := assertValNotNil(t, "", reflect.ValueOf(s)) + _, err = config.MultiErrorList(err) + assert.NoError(t, err) +} + +// assertFieldsNotNil recursively checks the struct s for nil fields. +func assertFieldsNotNil(t *testing.T, prefix string, s reflect.Value) (err error) { + t.Helper() + require.Equal(t, reflect.Struct, s.Kind()) + + typ := s.Type() + for i := 0; i < s.NumField(); i++ { + f := s.Field(i) + key := prefix + if tf := typ.Field(i); !tf.Anonymous { + if key != "" { + key += "." + } + key += tf.Name + } + err = errors.Join(err, assertValNotNil(t, key, f)) + } + return +} + +// assertValuesNotNil recursively checks the map m for nil values. +func assertValuesNotNil(t *testing.T, prefix string, m reflect.Value) (err error) { + t.Helper() + require.Equal(t, reflect.Map, m.Kind()) + if prefix != "" { + prefix += "." + } + + mi := m.MapRange() + for mi.Next() { + key := prefix + mi.Key().String() + err = errors.Join(err, assertValNotNil(t, key, mi.Value())) + } + return +} + +// assertElementsNotNil recursively checks the slice s for nil values. +func assertElementsNotNil(t *testing.T, prefix string, s reflect.Value) (err error) { + t.Helper() + require.Equal(t, reflect.Slice, s.Kind()) + + for i := 0; i < s.Len(); i++ { + err = errors.Join(err, assertValNotNil(t, prefix, s.Index(i))) + } + return +} + +var ( + textUnmarshaler encoding.TextUnmarshaler + textUnmarshalerType = reflect.TypeOf(&textUnmarshaler).Elem() +) + +// assertValNotNil recursively checks that val is not nil. val must be a struct, map, slice, or point to one. +func assertValNotNil(t *testing.T, key string, val reflect.Value) error { + t.Helper() + k := val.Kind() + switch k { //nolint:exhaustive + case reflect.Ptr: + if val.IsNil() { + return fmt.Errorf("%s: nil", key) + } + } + if k == reflect.Ptr { + if val.Type().Implements(textUnmarshalerType) { + return nil // skip values unmarshaled from strings + } + val = val.Elem() + } + switch val.Kind() { + case reflect.Struct: + if val.Type().Implements(textUnmarshalerType) { + return nil // skip values unmarshaled from strings + } + return assertFieldsNotNil(t, key, val) + case reflect.Map: + if val.IsNil() { + return nil // not actually a problem + } + return assertValuesNotNil(t, key, val) + case reflect.Slice: + if val.IsNil() { + return nil // not actually a problem + } + return assertElementsNotNil(t, key, val) + default: + return nil + } +} + +// AssertDocsTOMLComplete ensures that docsTOML contains every field in C and no extra fields. +func AssertDocsTOMLComplete[C any](t *testing.T, docsTOML string) { + t.Helper() + var c C + err := config.DecodeTOML(strings.NewReader(docsTOML), &c) + if err != nil && strings.Contains(err.Error(), "undecoded keys: ") { + t.Errorf("Docs contain extra fields: %v", err) + } else { + require.NoError(t, err) + } + AssertFieldsNotNil(t, c) +} + +// AssertFullMarshal ensures that c encodes to expTOML and contains no nil fields. +func AssertFullMarshal[C any](t *testing.T, c C, expTOML string) { + t.Helper() + AssertFieldsNotNil(t, c) + + b, err := toml.Marshal(c) + require.NoError(t, err) + s := string(b) + t.Log(s) + assert.Equal(t, expTOML, s, diff.Diff(expTOML, s)) +} diff --git a/pkg/config/configtest/defaults.go b/pkg/config/configtest/defaults.go new file mode 100644 index 0000000000..d0535a8529 --- /dev/null +++ b/pkg/config/configtest/defaults.go @@ -0,0 +1,85 @@ +package configtest + +import ( + "bufio" + "fmt" + "io" + "reflect" + "strings" +) + +// DocDefaultsOnly reads only the default values from a docs TOML file and decodes in to cfg. +// Fields without defaults will set to zero values. +func DocDefaultsOnly(r io.Reader, cfg any, decode func(io.Reader, any) error) error { + pr, pw := io.Pipe() + defer pr.Close() + go writeDefaults(r, pw) + if err := decode(pr, cfg); err != nil { + return fmt.Errorf("failed to decode default core configuration: %w", err) + } + // replace niled examples with zero values. + nilToZero(reflect.ValueOf(cfg)) + return nil +} + +// writeDefaults writes default lines from defaultsTOML to w. +func writeDefaults(r io.Reader, w *io.PipeWriter) { + defer w.Close() + s := bufio.NewScanner(r) + for s.Scan() { + t := s.Text() + // Skip comments and examples (which become zero values) + if strings.HasPrefix(t, "#") || strings.HasSuffix(t, "# Example") { + continue + } + if _, err := io.WriteString(w, t); err != nil { + w.CloseWithError(err) + } + if _, err := w.Write([]byte{'\n'}); err != nil { + w.CloseWithError(err) + } + } + if err := s.Err(); err != nil { + w.CloseWithError(fmt.Errorf("failed to scan core defaults: %w", err)) + } +} + +func nilToZero(val reflect.Value) { + if val.Kind() == reflect.Ptr { + if val.IsNil() { + t := val.Type().Elem() + val.Set(reflect.New(t)) + } + if val.Type().Implements(textUnmarshalerType) { + return // don't descend inside - leave whole zero value + } + val = val.Elem() + } + switch val.Kind() { + case reflect.Struct: + if val.Type().Implements(textUnmarshalerType) { + return // skip values unmarshaled from strings + } + for i := 0; i < val.NumField(); i++ { + f := val.Field(i) + nilToZero(f) + } + return + case reflect.Map: + if !val.IsNil() { + for _, k := range val.MapKeys() { + nilToZero(val.MapIndex(k)) + } + } + return + case reflect.Slice: + if !val.IsNil() { + for i := 0; i < val.Len(); i++ { + nilToZero(val.Index(i)) + } + } + return + default: + return + } +} diff --git a/pkg/config/error.go b/pkg/config/errors.go similarity index 99% rename from pkg/config/error.go rename to pkg/config/errors.go index dce631387f..ba3bef9577 100644 --- a/pkg/config/error.go +++ b/pkg/config/errors.go @@ -113,7 +113,7 @@ func (m multiErrorList) Unwrap() []error { return m } -// Flatten calls `Unwrap() []error` on each error and subsequent returned error that implement the method, returning a fully flattend sequence. +// Flatten calls `Unwrap() []error` on each error and subsequent returned error that implement the method, returning a fully flattened sequence. // //nolint:errorlint // error type checks will fail on wrapped errors. Disabled since we are not doing checks on error types. func Flatten(errs ...error) (flat []error) { diff --git a/pkg/sqlutil/sqltest/txdb.go b/pkg/sqlutil/sqltest/txdb.go index b3ee48de05..68a29ed706 100644 --- a/pkg/sqlutil/sqltest/txdb.go +++ b/pkg/sqlutil/sqltest/txdb.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "database/sql/driver" + "errors" "fmt" "io" "net/url" @@ -11,7 +12,6 @@ import ( "sync" "github.com/jmoiron/sqlx" - "go.uber.org/multierr" "github.com/smartcontractkit/chainlink-common/pkg/sqlutil/pg" "github.com/smartcontractkit/chainlink-common/pkg/utils" @@ -385,7 +385,7 @@ func (s stmt) Query(args []driver.Value) (driver.Rows, error) { } rows, err := s.st.Query(mapArgs(args)...) defer func() { - err = multierr.Combine(err, rows.Close()) + err = errors.Join(err, rows.Close()) }() if err != nil { return nil, err