feat: Add publish command (#1143)

As we are productionizing and rolling out our own registry we need to move from our format being documented in a yaml file inside goreleaser to being part of our code and part of the sdk.

Right now this just build the package without publishing as our backend is not ready but nailing down this will also help finalize the backend which Im working on as well.

The idea here is **not** to introduce a `package.json` file like  in npm and then create yet another language inside `json` or `yaml` config but rather the other way around and use all the information we already have from the plugin author and build a package with a manifest.json so everything is configurable in the code.

we will have two types of packages in our registry:
- `native` (for languages like Go/Rust/C)
- `docker` (for runtime language like .net, Java, Python)

For native this will looks the following:

```
ls -l ./dist
(base) yevgenyp@yevgenys-mbp aws % ls -l dist 
total 1142360
-rw-r--r--  1 yevgenyp  staff        447 Aug 13 20:08 plugin.json
-rw-r--r--  1 yevgenyp  staff  145537822 Aug 13 20:08 plugin_darwin_amd64.zip
-rw-r--r--  1 yevgenyp  staff  145537822 Aug 13 20:08 plugin_darwin_arm64.zip
-rw-r--r--  1 yevgenyp  staff  145537820 Aug 13 20:07 plugin_linux_amd64.zip
-rw-r--r--  1 yevgenyp  staff  145537824 Aug 13 20:07 plugin_windows_amd64.zip
-rw-r--r--  1 yevgenyp  staff    2726434 Aug 13 20:07 tables.json
(base) yevgenyp@yevgenys-mbp aws % 
```

manifest will look the following:

```
cat plugin.json
{
  "name": "aws",
  "version": "Development",
  "title": "aws",
  "short_description": "",
  "description": "",
  "categories": [],
  "protocols": [
    3
  ],
  "supported_targets": [
    {
      "os": "linux",
      "arch": "amd64"
    },
    {
      "os": "windows",
      "arch": "amd64"
    },
    {
      "os": "darwin",
      "arch": "amd64"
    },
    {
      "os": "darwin",
      "arch": "arm64"
    }
  ],
  "package_type": "native"
}
```

This will make sure the cloudquery package registry has all the information needed to publish it, display it on CloudQuery Hub and give it for client to download without running the plugins itself. Not running the plugin itself is important for the following reasons:
- Security: not running untrusted code in our environment.
- Developer experience: moving any issues closer to the developer and shortening the debugging/publishing loop. 

Once it gets a first review I'll also add tests.

Author: yevgenypats

examples/simple_plugin/.gitignore

@@ -0,0 +1 @@
+dist

examples/simple_plugin/client/client.go

@@ -0,0 +1,38 @@
+package client
+
+import (
+	"context"
+
+	"github.com/cloudquery/plugin-sdk/v4/schema"
+	"github.com/rs/zerolog"
+)
+
+type TestClient struct {
+	Logger   zerolog.Logger
+	Spec     Spec
+	ClientID int
+}
+
+func (*TestClient) ID() string {
+	return "TestClient"
+}
+
+func (c *TestClient) withClientID(i int) *TestClient {
+	t := *c
+	t.ClientID = i
+	return &t
+}
+
+func MultiplexBySpec(meta schema.ClientMeta) []schema.ClientMeta {
+	cl := meta.(*TestClient)
+	clients := make([]schema.ClientMeta, cl.Spec.NumClients)
+	for i := 0; i < cl.Spec.NumClients; i++ {
+		clients[i] = cl.withClientID(i)
+	}
+	return clients
+}
+
+func ResolveClientID(_ context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error {
+	cl := meta.(*TestClient)
+	return resource.Set(c.Name, cl.ClientID)
+}

examples/simple_plugin/client/spec.go

@@ -0,0 +1,15 @@
+package client
+
+type Spec struct {
+	NumClients int `json:"num_clients"`
+}
+
+func (s *Spec) SetDefaults() {
+	if s.NumClients <= 0 {
+		s.NumClients = 1
+	}
+}
+
+func (*Spec) Validate() error {
+	return nil
+}

examples/simple_plugin/go.mod

@@ -0,0 +1,64 @@
+module github.com/cloudquery/plugin-sdk/examples/simple_plugin
+
+go 1.20
+
+require (
+	github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882
+	github.com/cloudquery/plugin-sdk/v4 v4.4.0
+	github.com/rs/zerolog v1.30.0
+)
+
+replace github.com/cloudquery/plugin-sdk/v4 => ../../
+
+require (
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/cloudquery/plugin-pb-go v1.9.2 // indirect
+	github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/getsentry/sentry-go v0.20.0 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/flatbuffers v23.5.26+incompatible // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.3 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/pierrec/lz4/v4 v4.1.18 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/spf13/cobra v1.6.1 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.8.4 // indirect
+	github.com/thoas/go-funk v0.9.3 // indirect
+	github.com/zeebo/xxh3 v1.0.2 // indirect
+	go.opentelemetry.io/otel v1.16.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 // indirect
+	go.opentelemetry.io/otel/metric v1.16.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.16.0 // indirect
+	go.opentelemetry.io/otel/trace v1.16.0 // indirect
+	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
+	golang.org/x/exp v0.0.0-20230728194245-b0cb94b80691 // indirect
+	golang.org/x/mod v0.11.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
+	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
+	golang.org/x/tools v0.6.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/genproto v0.0.0-20230731193218-e0aa005b6bdf // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230731193218-e0aa005b6bdf // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230731193218-e0aa005b6bdf // indirect
+	google.golang.org/grpc v1.57.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

examples/simple_plugin/go.sum

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"context"
+	"log"
+
+	"github.com/cloudquery/plugin-sdk/examples/simple_plugin/plugin"
+	"github.com/cloudquery/plugin-sdk/v4/serve"
+)
+
+func main() {
+	p := serve.Plugin(plugin.Plugin())
+	if err := p.Serve(context.Background()); err != nil {
+		log.Fatalf("failed to serve plugin: %v", err)
+	}
+}

examples/simple_plugin/main.go

@@ -0,0 +1,96 @@
+package plugin
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/cloudquery/plugin-sdk/examples/simple_plugin/client"
+	"github.com/cloudquery/plugin-sdk/examples/simple_plugin/services"
+	"github.com/cloudquery/plugin-sdk/v4/message"
+	"github.com/cloudquery/plugin-sdk/v4/plugin"
+	"github.com/cloudquery/plugin-sdk/v4/scheduler"
+	"github.com/cloudquery/plugin-sdk/v4/schema"
+	"github.com/cloudquery/plugin-sdk/v4/transformers"
+	"github.com/rs/zerolog"
+)
+
+type Client struct {
+	logger    zerolog.Logger
+	config    client.Spec
+	tables    schema.Tables
+	scheduler *scheduler.Scheduler
+
+	plugin.UnimplementedDestination
+}
+
+func (c *Client) Logger() *zerolog.Logger {
+	return &c.logger
+}
+
+func (c *Client) Sync(ctx context.Context, options plugin.SyncOptions, res chan<- message.SyncMessage) error {
+	tt, err := c.tables.FilterDfs(options.Tables, options.SkipTables, options.SkipDependentTables)
+	if err != nil {
+		return err
+	}
+
+	schedulerClient := &client.TestClient{
+		Logger: c.logger,
+		Spec:   c.config,
+	}
+
+	return c.scheduler.Sync(ctx, schedulerClient, tt, res, scheduler.WithSyncDeterministicCQID(options.DeterministicCQID))
+}
+
+func (c *Client) Tables(_ context.Context, options plugin.TableOptions) (schema.Tables, error) {
+	tt, err := c.tables.FilterDfs(options.Tables, options.SkipTables, options.SkipDependentTables)
+	if err != nil {
+		return nil, err
+	}
+
+	return tt, nil
+}
+
+func (*Client) Close(_ context.Context) error {
+	return nil
+}
+
+func Configure(_ context.Context, logger zerolog.Logger, spec []byte, opts plugin.NewClientOptions) (plugin.Client, error) {
+	if opts.NoConnection {
+		return &Client{
+			logger: logger,
+			tables: getTables(),
+		}, nil
+	}
+
+	config := &client.Spec{}
+	if err := json.Unmarshal(spec, config); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal spec: %w", err)
+	}
+	config.SetDefaults()
+	if err := config.Validate(); err != nil {
+		return nil, fmt.Errorf("failed to validate spec: %w", err)
+	}
+
+	return &Client{
+		logger: logger,
+		config: *config,
+		scheduler: scheduler.NewScheduler(
+			scheduler.WithLogger(logger),
+		),
+		tables: getTables(),
+	}, nil
+}
+
+func getTables() schema.Tables {
+	tables := schema.Tables{
+		services.TestSomeTable(),
+	}
+	if err := transformers.TransformTables(tables); err != nil {
+		panic(err)
+	}
+	for _, t := range tables {
+		schema.AddCqIDs(t)
+	}
+	return tables
+}

examples/simple_plugin/plugin/client.go

@@ -0,0 +1,17 @@
+package plugin
+
+import (
+	"github.com/cloudquery/plugin-sdk/v4/plugin"
+)
+
+var (
+	Version = "development"
+)
+
+func Plugin() *plugin.Plugin {
+	return plugin.NewPlugin(
+		"test",
+		Version,
+		Configure,
+	)
+}

examples/simple_plugin/plugin/plugin.go

@@ -0,0 +1,48 @@
+package services
+
+import (
+	"context"
+
+	"github.com/apache/arrow/go/v13/arrow"
+	"github.com/cloudquery/plugin-sdk/examples/simple_plugin/client"
+	"github.com/cloudquery/plugin-sdk/v4/schema"
+)
+
+func TestSomeTable() *schema.Table {
+	return &schema.Table{
+		Name:        "test_some_table",
+		Description: "Test description",
+		Resolver:    fetchSomeTableData,
+		Multiplex:   client.MultiplexBySpec,
+		Columns: []schema.Column{
+			{
+				Name:        "column1",
+				Description: "Test Column 1",
+				Type:        arrow.BinaryTypes.String,
+				PrimaryKey:  true,
+				Resolver:    schema.PathResolver("column1"),
+			},
+			{
+				Name:        "column2",
+				Description: "Test Column 2",
+				Type:        arrow.PrimitiveTypes.Int64,
+				Resolver:    schema.PathResolver("column2"),
+			},
+			{
+				Name:        "client_id",
+				Description: "ID of client",
+				Type:        arrow.PrimitiveTypes.Int64,
+				Resolver:    client.ResolveClientID,
+				PrimaryKey:  true,
+			},
+		},
+	}
+}
+
+func fetchSomeTableData(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
+	res <- map[string]any{
+		"column1": "test_project_id",
+		"column2": 123,
+	}
+	return nil
+}