fix: Make gen docs work without auth (#1052)

yevgenypats · web-flow · commit 504f8498f9f1 · 2023-07-03T15:06:20.000Z
&lt;!--
Explain what problem this PR addresses
--&gt;

---
diff --git a/docs/templates/table.md.go.tpl b/docs/templates/table.md.go.tpl
@@ -40,5 +40,5 @@ The following tables depend on {{.Name}}:
 | Name          | Type          |
 | ------------- | ------------- |
 {{- range $column := $.Columns }}
-|{{$column.Name}}{{if $column.PrimaryKey}} (PK){{end}}{{if $column.IncrementalKey}} (Incremental Key){{end}}|{{$column.Type}}|
+|{{$column.Name}}{{if $column.PrimaryKey}} (PK){{end}}{{if $column.IncrementalKey}} (Incremental Key){{end}}|`{{$column.Type}}`|
 {{- end }}
diff --git a/docs/testdata/TestGeneratePluginDocs-Markdown-incremental_table.md b/docs/testdata/TestGeneratePluginDocs-Markdown-incremental_table.md
@@ -11,6 +11,6 @@ It supports incremental syncs based on the (**id_col**, **id_col2**) columns.
 
 | Name          | Type          |
 | ------------- | ------------- |
-|int_col|int64|
-|id_col (PK) (Incremental Key)|int64|
-|id_col2 (Incremental Key)|int64|
+|int_col|`int64`|
+|id_col (PK) (Incremental Key)|`int64`|
+|id_col2 (Incremental Key)|`int64`|
diff --git a/docs/testdata/TestGeneratePluginDocs-Markdown-relation_relation_table_a.md b/docs/testdata/TestGeneratePluginDocs-Markdown-relation_relation_table_a.md
@@ -14,4 +14,4 @@ This table depends on [relation_table](relation_table.md).
 
 | Name          | Type          |
 | ------------- | ------------- |
-|string_col|utf8|
+|string_col|`utf8`|
diff --git a/docs/testdata/TestGeneratePluginDocs-Markdown-relation_relation_table_b.md b/docs/testdata/TestGeneratePluginDocs-Markdown-relation_relation_table_b.md
@@ -14,4 +14,4 @@ This table depends on [relation_table](relation_table.md).
 
 | Name          | Type          |
 | ------------- | ------------- |
-|string_col|utf8|
+|string_col|`utf8`|
diff --git a/docs/testdata/TestGeneratePluginDocs-Markdown-relation_table.md b/docs/testdata/TestGeneratePluginDocs-Markdown-relation_table.md
@@ -18,4 +18,4 @@ The following tables depend on relation_table:
 
 | Name          | Type          |
 | ------------- | ------------- |
-|string_col|utf8|
+|string_col|`utf8`|
diff --git a/docs/testdata/TestGeneratePluginDocs-Markdown-test_table.md b/docs/testdata/TestGeneratePluginDocs-Markdown-test_table.md
@@ -16,10 +16,10 @@ The following tables depend on test_table:
 
 | Name          | Type          |
 | ------------- | ------------- |
-|int_col|int64|
-|id_col (PK)|int64|
-|id_col2 (PK)|int64|
-|json_col|json|
-|list_col|list<item: int64, nullable>|
-|map_col|map<utf8, int64, items_nullable>|
-|struct_col|struct<string_field: utf8, int_field: int64>|
+|int_col|`int64`|
+|id_col (PK)|`int64`|
+|id_col2 (PK)|`int64`|
+|json_col|`json`|
+|list_col|`list<item: int64, nullable>`|
+|map_col|`map<utf8, int64, items_nullable>`|
+|struct_col|`struct<string_field: utf8, int_field: int64>`|
diff --git a/internal/memdb/memdb.go b/internal/memdb/memdb.go
@@ -48,16 +48,16 @@ func GetNewClient(options ...Option) plugin.NewClientFunc {
 	for _, opt := range options {
 		opt(c)
 	}
-	return func(context.Context, zerolog.Logger, []byte) (plugin.Client, error) {
+	return func(context.Context, zerolog.Logger, []byte, plugin.NewClientOptions) (plugin.Client, error) {
 		return c, nil
 	}
 }
 
-func NewMemDBClient(ctx context.Context, l zerolog.Logger, spec []byte) (plugin.Client, error) {
-	return GetNewClient()(ctx, l, spec)
+func NewMemDBClient(ctx context.Context, l zerolog.Logger, spec []byte, options plugin.NewClientOptions) (plugin.Client, error) {
+	return GetNewClient()(ctx, l, spec, options)
 }
 
-func NewMemDBClientErrOnNew(context.Context, zerolog.Logger, []byte) (plugin.Client, error) {
+func NewMemDBClientErrOnNew(context.Context, zerolog.Logger, []byte, plugin.NewClientOptions) (plugin.Client, error) {
 	return nil, fmt.Errorf("newTestDestinationMemDBClientErrOnNew")
 }
 
@@ -123,7 +123,7 @@ func (c *client) Sync(_ context.Context, options plugin.SyncOptions, res chan<-
 	return nil
 }
 
-func (c *client) Tables(_ context.Context) (schema.Tables, error) {
+func (c *client) Tables(context.Context, plugin.TableOptions) (schema.Tables, error) {
 	tables := make(schema.Tables, 0, len(c.tables))
 	for _, table := range c.tables {
 		tables = append(tables, table)
diff --git a/internal/memdb/memdb_test.go b/internal/memdb/memdb_test.go
@@ -10,7 +10,7 @@ import (
 func TestPlugin(t *testing.T) {
 	ctx := context.Background()
 	p := plugin.NewPlugin("test", "development", NewMemDBClient)
-	if err := p.Init(ctx, nil); err != nil {
+	if err := p.Init(ctx, nil, plugin.NewClientOptions{}); err != nil {
 		t.Fatal(err)
 	}
 	plugin.TestWriterSuiteRunner(
@@ -25,7 +25,7 @@ func TestPlugin(t *testing.T) {
 func TestPluginOnNewError(t *testing.T) {
 	ctx := context.Background()
 	p := plugin.NewPlugin("test", "development", NewMemDBClientErrOnNew)
-	err := p.Init(ctx, nil)
+	err := p.Init(ctx, nil, plugin.NewClientOptions{})
 
 	if err == nil {
 		t.Fatal("expected error")
@@ -36,7 +36,7 @@ func TestOnWriteError(t *testing.T) {
 	ctx := context.Background()
 	newClientFunc := GetNewClient(WithErrOnWrite())
 	p := plugin.NewPlugin("test", "development", newClientFunc)
-	if err := p.Init(ctx, nil); err != nil {
+	if err := p.Init(ctx, nil, plugin.NewClientOptions{}); err != nil {
 		t.Fatal(err)
 	}
 	if err := p.WriteAll(ctx, nil); err.Error() != "errOnWrite" {
diff --git a/internal/servers/destination/v0/destinations.go b/internal/servers/destination/v0/destinations.go
@@ -44,7 +44,7 @@ func (s *Server) Configure(ctx context.Context, req *pbBase.Configure_Request) (
 	if err != nil {
 		return nil, status.Errorf(codes.InvalidArgument, "failed to marshal spec: %v", err)
 	}
-	return &pbBase.Configure_Response{}, s.Plugin.Init(ctx, pluginSpec)
+	return &pbBase.Configure_Response{}, s.Plugin.Init(ctx, pluginSpec, plugin.NewClientOptions{})
 }
 
 func (s *Server) GetName(context.Context, *pbBase.GetName_Request) (*pbBase.GetName_Response, error) {
diff --git a/internal/servers/destination/v1/destinations.go b/internal/servers/destination/v1/destinations.go
@@ -54,7 +54,7 @@ func (s *Server) Configure(ctx context.Context, req *pb.Configure_Request) (*pb.
 	if err != nil {
 		return nil, status.Errorf(codes.InvalidArgument, "failed to marshal spec: %v", err)
 	}
-	return &pb.Configure_Response{}, s.Plugin.Init(ctx, pluginSpec)
+	return &pb.Configure_Response{}, s.Plugin.Init(ctx, pluginSpec, plugin.NewClientOptions{})
 }
 
 func (s *Server) GetName(context.Context, *pb.GetName_Request) (*pb.GetName_Response, error) {
diff --git a/internal/servers/plugin/v3/plugin.go b/internal/servers/plugin/v3/plugin.go
@@ -27,8 +27,11 @@ type Server struct {
 	NoSentry  bool
 }
 
-func (s *Server) GetTables(ctx context.Context, _ *pb.GetTables_Request) (*pb.GetTables_Response, error) {
-	tables, err := s.Plugin.Tables(ctx)
+func (s *Server) GetTables(ctx context.Context, req *pb.GetTables_Request) (*pb.GetTables_Response, error) {
+	tables, err := s.Plugin.Tables(ctx, plugin.TableOptions{
+		Tables:     req.Tables,
+		SkipTables: req.SkipTables,
+	})
 	if err != nil {
 		return nil, status.Errorf(codes.Internal, "failed to get tables: %v", err)
 	}
@@ -58,7 +61,7 @@ func (s *Server) GetVersion(context.Context, *pb.GetVersion_Request) (*pb.GetVer
 }
 
 func (s *Server) Init(ctx context.Context, req *pb.Init_Request) (*pb.Init_Response, error) {
-	if err := s.Plugin.Init(ctx, req.Spec); err != nil {
+	if err := s.Plugin.Init(ctx, req.Spec, plugin.NewClientOptions{}); err != nil {
 		return nil, status.Errorf(codes.Internal, "failed to init plugin: %v", err)
 	}
 	return &pb.Init_Response{}, nil
diff --git a/plugin/options.go b/plugin/options.go
@@ -16,3 +16,9 @@ func (m MigrateMode) String() string {
 }
 
 type Option func(*Plugin)
+
+type TableOptions struct {
+	Tables              []string
+	SkipTables          []string
+	SkipDependentTables bool
+}
diff --git a/plugin/plugin.go b/plugin/plugin.go
@@ -13,7 +13,11 @@ import (
 
 var ErrNotImplemented = fmt.Errorf("not implemented")
 
-type NewClientFunc func(context.Context, zerolog.Logger, []byte) (Client, error)
+type NewClientOptions struct {
+	NoConnection bool
+}
+
+type NewClientFunc func(context.Context, zerolog.Logger, []byte, NewClientOptions) (Client, error)
 
 type Client interface {
 	SourceClient
@@ -36,7 +40,7 @@ func (UnimplementedSource) Sync(context.Context, SyncOptions, chan<- message.Syn
 	return ErrNotImplemented
 }
 
-func (UnimplementedSource) Tables(context.Context) (schema.Tables, error) {
+func (UnimplementedSource) Tables(context.Context, TableOptions) (schema.Tables, error) {
 	return nil, ErrNotImplemented
 }
 
@@ -91,25 +95,25 @@ func (p *Plugin) SetLogger(logger zerolog.Logger) {
 	p.logger = logger.With().Str("module", p.name+"-src").Logger()
 }
 
-func (p *Plugin) Tables(ctx context.Context) (schema.Tables, error) {
+func (p *Plugin) Tables(ctx context.Context, options TableOptions) (schema.Tables, error) {
 	if p.client == nil {
 		return nil, fmt.Errorf("plugin not initialized")
 	}
-	tables, err := p.client.Tables(ctx)
+	tables, err := p.client.Tables(ctx, options)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get tables: %w", err)
 	}
 	return tables, nil
 }
 
 // Init initializes the plugin with the given spec.
-func (p *Plugin) Init(ctx context.Context, spec []byte) error {
+func (p *Plugin) Init(ctx context.Context, spec []byte, options NewClientOptions) error {
 	if !p.mu.TryLock() {
 		return fmt.Errorf("plugin already in use")
 	}
 	defer p.mu.Unlock()
 	var err error
-	p.client, err = p.newClient(ctx, p.logger, spec)
+	p.client, err = p.newClient(ctx, p.logger, spec, options)
 	if err != nil {
 		return fmt.Errorf("failed to initialize client: %w", err)
 	}
diff --git a/plugin/plugin_source.go b/plugin/plugin_source.go
@@ -26,7 +26,7 @@ type SyncOptions struct {
 
 type SourceClient interface {
 	Close(ctx context.Context) error
-	Tables(ctx context.Context) (schema.Tables, error)
+	Tables(ctx context.Context, options TableOptions) (schema.Tables, error)
 	Sync(ctx context.Context, options SyncOptions, res chan<- message.SyncMessage) error
 }
 
@@ -49,7 +49,7 @@ type NewSourceClientFunc func(context.Context, zerolog.Logger, any) (SourceClien
 // NewSourcePlugin returns a new CloudQuery Plugin with the given name, version and implementation.
 // Source plugins only support read operations. For Read & Write plugin use NewPlugin.
 func NewSourcePlugin(name string, version string, newClient NewSourceClientFunc, options ...Option) *Plugin {
-	newClientWrapper := func(ctx context.Context, logger zerolog.Logger, spec []byte) (Client, error) {
+	newClientWrapper := func(ctx context.Context, logger zerolog.Logger, spec []byte, options NewClientOptions) (Client, error) {
 		sourceClient, err := newClient(ctx, logger, spec)
 		if err != nil {
 			return nil, err
diff --git a/plugin/plugin_test.go b/plugin/plugin_test.go
@@ -14,15 +14,15 @@ type testPluginClient struct {
 	messages message.SyncMessages
 }
 
-func newTestPluginClient(context.Context, zerolog.Logger, []byte) (Client, error) {
+func newTestPluginClient(context.Context, zerolog.Logger, []byte, NewClientOptions) (Client, error) {
 	return &testPluginClient{}, nil
 }
 
 func (*testPluginClient) GetSpec() any {
 	return &struct{}{}
 }
 
-func (*testPluginClient) Tables(context.Context) (schema.Tables, error) {
+func (*testPluginClient) Tables(context.Context, TableOptions) (schema.Tables, error) {
 	return schema.Tables{}, nil
 }
 
@@ -60,10 +60,10 @@ func (*testPluginClient) Close(context.Context) error {
 func TestPluginSuccess(t *testing.T) {
 	ctx := context.Background()
 	p := NewPlugin("test", "v1.0.0", newTestPluginClient)
-	if err := p.Init(ctx, []byte("")); err != nil {
+	if err := p.Init(ctx, []byte(""), NewClientOptions{}); err != nil {
 		t.Fatal(err)
 	}
-	tables, err := p.Tables(ctx)
+	tables, err := p.Tables(ctx, TableOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/serve/docs.go b/serve/docs.go
@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	"github.com/cloudquery/plugin-sdk/v4/docs"
+	"github.com/cloudquery/plugin-sdk/v4/plugin"
 	"github.com/spf13/cobra"
 )
 
@@ -30,7 +31,14 @@ func (s *PluginServe) newCmdPluginDoc() *cobra.Command {
 		Long:  pluginDocLong,
 		Args:  cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			tables, err := s.plugin.Tables(cmd.Context())
+			if err := s.plugin.Init(cmd.Context(), nil, plugin.NewClientOptions{
+				NoConnection: true,
+			}); err != nil {
+				return err
+			}
+			tables, err := s.plugin.Tables(cmd.Context(), plugin.TableOptions{
+				Tables: []string{"*"},
+			})
 			if err != nil {
 				return err
 			}
diff --git a/serve/docs_test.go b/serve/docs_test.go
@@ -1,7 +1,6 @@
 package serve
 
 import (
-	"context"
 	"testing"
 
 	"github.com/cloudquery/plugin-sdk/v4/internal/memdb"
@@ -14,9 +13,6 @@ func TestPluginDocs(t *testing.T) {
 		"testPlugin",
 		"v1.0.0",
 		memdb.NewMemDBClient)
-	if err := p.Init(context.Background(), nil); err != nil {
-		t.Fatal(err)
-	}
 	srv := Plugin(p)
 	cmd := srv.newCmdPluginRoot()
 	cmd.SetArgs([]string{"doc", tmpDir})

Original file line number	Diff line number	Diff line change
`@@ -48,16 +48,16 @@ func GetNewClient(options ...Option) plugin.NewClientFunc {`
`48`	`48`	`for _, opt := range options {`
`49`	`49`	`opt(c)`
`50`	`50`	`}`
`51`		`- return func(context.Context, zerolog.Logger, []byte) (plugin.Client, error) {`
	`51`	`+ return func(context.Context, zerolog.Logger, []byte, plugin.NewClientOptions) (plugin.Client, error) {`
`52`	`52`	`return c, nil`
`53`	`53`	`}`
`54`	`54`	`}`
`55`	`55`
`56`		`-func NewMemDBClient(ctx context.Context, l zerolog.Logger, spec []byte) (plugin.Client, error) {`
`57`		`- return GetNewClient()(ctx, l, spec)`
	`56`	`+func NewMemDBClient(ctx context.Context, l zerolog.Logger, spec []byte, options plugin.NewClientOptions) (plugin.Client, error) {`
	`57`	`+ return GetNewClient()(ctx, l, spec, options)`
`58`	`58`	`}`
`59`	`59`
`60`		`-func NewMemDBClientErrOnNew(context.Context, zerolog.Logger, []byte) (plugin.Client, error) {`
	`60`	`+func NewMemDBClientErrOnNew(context.Context, zerolog.Logger, []byte, plugin.NewClientOptions) (plugin.Client, error) {`
`61`	`61`	`return nil, fmt.Errorf("newTestDestinationMemDBClientErrOnNew")`
`62`	`62`	`}`
`63`	`63`
`@@ -123,7 +123,7 @@ func (c *client) Sync(_ context.Context, options plugin.SyncOptions, res chan<-`
`123`	`123`	`return nil`
`124`	`124`	`}`
`125`	`125`
`126`		`-func (c *client) Tables(_ context.Context) (schema.Tables, error) {`
	`126`	`+func (c *client) Tables(context.Context, plugin.TableOptions) (schema.Tables, error) {`
`127`	`127`	`tables := make(schema.Tables, 0, len(c.tables))`
`128`	`128`	`for _, table := range c.tables {`
`129`	`129`	`tables = append(tables, table)`
Original file line number	Diff line number	Diff line change
`@@ -44,7 +44,7 @@ func (s Server) Configure(ctx context.Context, req pbBase.Configure_Request) (`
`44`	`44`	`if err != nil {`
`45`	`45`	`return nil, status.Errorf(codes.InvalidArgument, "failed to marshal spec: %v", err)`
`46`	`46`	`}`
`47`		`- return &pbBase.Configure_Response{}, s.Plugin.Init(ctx, pluginSpec)`
	`47`	`+ return &pbBase.Configure_Response{}, s.Plugin.Init(ctx, pluginSpec, plugin.NewClientOptions{})`
`48`	`48`	`}`
`49`	`49`
`50`	`50`	`func (s Server) GetName(context.Context, pbBase.GetName_Request) (*pbBase.GetName_Response, error) {`
Original file line number	Diff line number	Diff line change
`@@ -54,7 +54,7 @@ func (s Server) Configure(ctx context.Context, req pb.Configure_Request) (*pb.`
`54`	`54`	`if err != nil {`
`55`	`55`	`return nil, status.Errorf(codes.InvalidArgument, "failed to marshal spec: %v", err)`
`56`	`56`	`}`
`57`		`- return &pb.Configure_Response{}, s.Plugin.Init(ctx, pluginSpec)`
	`57`	`+ return &pb.Configure_Response{}, s.Plugin.Init(ctx, pluginSpec, plugin.NewClientOptions{})`
`58`	`58`	`}`
`59`	`59`
`60`	`60`	`func (s Server) GetName(context.Context, pb.GetName_Request) (*pb.GetName_Response, error) {`