feat: Move specs from SDK

Author: yevgenypats

go.mod

@@ -3,14 +3,22 @@ module github.com/cloudquery/plugin-pb-go
 go 1.19
 
 require (
+	github.com/ghodss/yaml v1.0.0
+	github.com/google/go-cmp v0.5.9
+	github.com/stretchr/testify v1.4.0
+	github.com/thoas/go-funk v0.9.3
 	google.golang.org/grpc v1.54.0
 	google.golang.org/protobuf v1.28.1
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
+	github.com/davecgh/go-spew v1.1.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	golang.org/x/net v0.8.0 // indirect
 	golang.org/x/sys v0.6.0 // indirect
 	golang.org/x/text v0.8.0 // indirect
 	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go.sum

@@ -1,8 +1,20 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw=
+github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
 golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
@@ -18,3 +30,10 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

specs/backend.go

@@ -0,0 +1,60 @@
+package specs
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+)
+
+type Backend int
+
+const (
+	BackendNone Backend = iota
+	BackendLocal
+)
+
+var AllBackends = Backends{BackendNone, BackendLocal}
+var AllBackendNames = [...]string{
+	BackendNone:  "none",
+	BackendLocal: "local",
+}
+
+type Backends []Backend
+
+func (s Backends) String() string {
+	var buffer bytes.Buffer
+	for i, backend := range s {
+		if i > 0 {
+			buffer.WriteString(", ")
+		}
+		buffer.WriteString(backend.String())
+	}
+	return buffer.String()
+}
+
+func (s Backend) String() string {
+	return AllBackendNames[s]
+}
+func (s Backend) MarshalJSON() ([]byte, error) {
+	return []byte(`"` + s.String() + `"`), nil
+}
+
+func (s *Backend) UnmarshalJSON(data []byte) (err error) {
+	var backend string
+	if err := json.Unmarshal(data, &backend); err != nil {
+		return err
+	}
+	if *s, err = BackendFromString(backend); err != nil {
+		return err
+	}
+	return nil
+}
+
+func BackendFromString(s string) (Backend, error) {
+	for i, backend := range AllBackendNames {
+		if s == backend {
+			return Backend(i), nil
+		}
+	}
+	return BackendNone, fmt.Errorf("unknown backend %s", s)
+}

specs/backend_test.go

@@ -0,0 +1,36 @@
+package specs
+
+import (
+	"encoding/json"
+	"testing"
+
+	"gopkg.in/yaml.v3"
+)
+
+func TestBackendJsonMarshalUnmarshal(t *testing.T) {
+	b, err := json.Marshal(BackendLocal)
+	if err != nil {
+		t.Fatal("failed to marshal backend:", err)
+	}
+	var backend Backend
+	if err := json.Unmarshal(b, &backend); err != nil {
+		t.Fatal("failed to unmarshal backend:", err)
+	}
+	if backend != BackendLocal {
+		t.Fatal("expected backend to be local, but got:", backend)
+	}
+}
+
+func TestBackendYamlMarshalUnmarshal(t *testing.T) {
+	b, err := yaml.Marshal(BackendLocal)
+	if err != nil {
+		t.Fatal("failed to marshal backend:", err)
+	}
+	var backend Backend
+	if err := yaml.Unmarshal(b, &backend); err != nil {
+		t.Fatal("failed to unmarshal backend:", err)
+	}
+	if backend != BackendLocal {
+		t.Fatal("expected backend to be local, but got:", backend)
+	}
+}

specs/destination.go

@@ -0,0 +1,88 @@
+package specs
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/thoas/go-funk"
+)
+
+type Destination struct {
+	Name           string      `json:"name,omitempty"`
+	Version        string      `json:"version,omitempty"`
+	Path           string      `json:"path,omitempty"`
+	Registry       Registry    `json:"registry,omitempty"`
+	WriteMode      WriteMode   `json:"write_mode,omitempty"`
+	MigrateMode    MigrateMode `json:"migrate_mode,omitempty"`
+	BatchSize      int         `json:"batch_size,omitempty"`
+	BatchSizeBytes int         `json:"batch_size_bytes,omitempty"`
+	Spec           any         `json:"spec,omitempty"`
+	PKMode         PKMode      `json:"pk_mode,omitempty"`
+}
+
+func (d *Destination) SetDefaults(defaultBatchSize, defaultBatchSizeBytes int) {
+	if d.Registry.String() == "" {
+		d.Registry = RegistryGithub
+	}
+	if d.BatchSize == 0 {
+		d.BatchSize = defaultBatchSize
+	}
+	if d.BatchSizeBytes == 0 {
+		d.BatchSizeBytes = defaultBatchSizeBytes
+	}
+}
+
+func (d *Destination) UnmarshalSpec(out any) error {
+	b, err := json.Marshal(d.Spec)
+	if err != nil {
+		return err
+	}
+	dec := json.NewDecoder(bytes.NewReader(b))
+	dec.UseNumber()
+	dec.DisallowUnknownFields()
+	return dec.Decode(out)
+}
+
+func (d *Destination) Validate() error {
+	if d.Name == "" {
+		return fmt.Errorf("name is required")
+	}
+	if d.Path == "" {
+		msg := "path is required"
+		// give a small hint to help users transition from the old config format that didn't require path
+		officialPlugins := []string{"postgresql", "csv"}
+		if funk.ContainsString(officialPlugins, d.Name) {
+			msg += fmt.Sprintf(". Hint: try setting path to cloudquery/%s in your config", d.Name)
+		}
+		return fmt.Errorf(msg)
+	}
+
+	if d.Registry == RegistryGithub {
+		if d.Version == "" {
+			return fmt.Errorf("version is required")
+		}
+		if !strings.HasPrefix(d.Version, "v") {
+			return fmt.Errorf("version must start with v")
+		}
+	}
+	if d.BatchSize < 0 {
+		return fmt.Errorf("batch_size must be greater than 0")
+	}
+	return nil
+}
+
+func (d Destination) VersionString() string {
+	if d.Registry != RegistryGithub {
+		return fmt.Sprintf("%s (%s@%s)", d.Name, d.Registry, d.Path)
+	}
+	pathParts := strings.Split(d.Path, "/")
+	if len(pathParts) != 2 {
+		return fmt.Sprintf("%s (%s@%s)", d.Name, d.Path, d.Version)
+	}
+	if d.Name == pathParts[1] {
+		return fmt.Sprintf("%s (%s)", d.Name, d.Version)
+	}
+	return fmt.Sprintf("%s (%s@%s)", d.Name, pathParts[1], d.Version)
+}