feat!: Update to SDK V4 (#984)

Instead of this #915 (fix
rebase conflicts)

This introduce a single API for both destination and sources. Now
plugins should be able to both read and write.
This also goes together with a proto upgrade
https://github.com/cloudquery/plugin-pb-go/tree/main/pb/plugin/v3

---------

Co-authored-by: Herman Schaaf <[email protected]>
Co-authored-by: Kemal Hadimli <[email protected]>
Co-authored-by: Erez Rokah <[email protected]>

Author: 4 people

.github/workflows/lint_markdown.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Vale
         uses: errata-ai/vale-action@v2
         with:
-          vale_flags: "--glob=!{plugins/source/testdata/*,CHANGELOG.md,.github/styles/proselint/README.md}"
+          vale_flags: "--glob=!{docs/testdata/*,CHANGELOG.md,.github/styles/proselint/README.md}"
           filter_mode: nofilter
         env:
           GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
@@ -31,4 +31,4 @@ jobs:
         with:
           files: .
           config_file: .markdownlint.yaml
-          ignore_files: "{plugins/source/testdata/*,CHANGELOG.md}"
+          ignore_files: "{docs/testdata/*,CHANGELOG.md}"

.gitignore

@@ -8,4 +8,6 @@ config.hcl
 vendor
 cover.out
 .delta.*
-bench.json
+bench.json
+serve/^TestPluginDocs$/
+cover

backend/backend.go

@@ -0,0 +1,137 @@
+package docs
+
+import (
+	"embed"
+	"fmt"
+	"os"
+	"regexp"
+	"sort"
+
+	"github.com/cloudquery/plugin-sdk/v4/caser"
+	"github.com/cloudquery/plugin-sdk/v4/schema"
+)
+
+//go:embed templates/*.go.tpl
+var templatesFS embed.FS
+
+var reMatchNewlines = regexp.MustCompile(`\n{3,}`)
+var reMatchHeaders = regexp.MustCompile(`(#{1,6}.+)\n+`)
+
+var DefaultTitleExceptions = map[string]string{
+	// common abbreviations
+	"acl":   "ACL",
+	"acls":  "ACLs",
+	"api":   "API",
+	"apis":  "APIs",
+	"ca":    "CA",
+	"cidr":  "CIDR",
+	"cidrs": "CIDRs",
+	"db":    "DB",
+	"dbs":   "DBs",
+	"dhcp":  "DHCP",
+	"iam":   "IAM",
+	"iot":   "IOT",
+	"ip":    "IP",
+	"ips":   "IPs",
+	"ipv4":  "IPv4",
+	"ipv6":  "IPv6",
+	"mfa":   "MFA",
+	"ml":    "ML",
+	"oauth": "OAuth",
+	"vpc":   "VPC",
+	"vpcs":  "VPCs",
+	"vpn":   "VPN",
+	"vpns":  "VPNs",
+	"waf":   "WAF",
+	"wafs":  "WAFs",
+
+	// cloud providers
+	"aws": "AWS",
+	"gcp": "GCP",
+}
+
+type Format int
+
+const (
+	FormatMarkdown Format = iota
+	FormatJSON
+)
+
+func (r Format) String() string {
+	return [...]string{"markdown", "json"}[r]
+}
+
+func FormatFromString(s string) (Format, error) {
+	switch s {
+	case "markdown":
+		return FormatMarkdown, nil
+	case "json":
+		return FormatJSON, nil
+	default:
+		return FormatMarkdown, fmt.Errorf("unknown format %s", s)
+	}
+}
+
+type Generator struct {
+	tables           schema.Tables
+	titleTransformer func(*schema.Table) string
+	pluginName       string
+}
+
+func DefaultTitleTransformer(table *schema.Table) string {
+	if table.Title != "" {
+		return table.Title
+	}
+	csr := caser.New(caser.WithCustomExceptions(DefaultTitleExceptions))
+	return csr.ToTitle(table.Name)
+}
+
+func sortTables(tables schema.Tables) {
+	sort.SliceStable(tables, func(i, j int) bool {
+		return tables[i].Name < tables[j].Name
+	})
+
+	for _, table := range tables {
+		sortTables(table.Relations)
+	}
+}
+
+// NewGenerator creates a new generator for the given tables.
+// The tables are sorted by name. pluginName is optional and is used in markdown only
+func NewGenerator(pluginName string, tables schema.Tables) *Generator {
+	sortedTables := make(schema.Tables, 0, len(tables))
+	for _, t := range tables {
+		sortedTables = append(sortedTables, t.Copy(nil))
+	}
+	sortTables(sortedTables)
+
+	return &Generator{
+		tables:           sortedTables,
+		titleTransformer: DefaultTitleTransformer,
+		pluginName:       pluginName,
+	}
+}
+
+func (g *Generator) Generate(dir string, format Format) error {
+	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
+		return err
+	}
+
+	switch format {
+	case FormatMarkdown:
+		return g.renderTablesAsMarkdown(dir)
+	case FormatJSON:
+		return g.renderTablesAsJSON(dir)
+	default:
+		return fmt.Errorf("unsupported format: %v", format)
+	}
+}
+
+// setDestinationManagedCqColumns overwrites or adds the CQ columns that are managed by the destination plugins (_cq_sync_time, _cq_source_name).
+// func setDestinationManagedCqColumns(tables []*schema.Table) {
+// 	for _, table := range tables {
+// 		table.OverwriteOrAddColumn(&schema.CqSyncTimeColumn)
+// 		table.OverwriteOrAddColumn(&schema.CqSourceNameColumn)
+// 		setDestinationManagedCqColumns(table.Relations)
+// 	}
+// }

buf.yaml

@@ -1,6 +1,6 @@
 //go:build !windows
 
-package source
+package docs
 
 import (
 	"os"
@@ -9,8 +9,8 @@ import (
 
 	"github.com/apache/arrow/go/v13/arrow"
 	"github.com/bradleyjkemp/cupaloy/v2"
-	"github.com/cloudquery/plugin-sdk/v3/schema"
-	"github.com/cloudquery/plugin-sdk/v3/types"
+	"github.com/cloudquery/plugin-sdk/v4/schema"
+	"github.com/cloudquery/plugin-sdk/v4/types"
 	"github.com/stretchr/testify/require"
 )
 
@@ -120,14 +120,13 @@ var testTables = []*schema.Table{
 }
 
 func TestGeneratePluginDocs(t *testing.T) {
-	p := NewPlugin("test", "v1.0.0", testTables, newTestExecutionClient)
-
+	g := NewGenerator("test", testTables)
 	cup := cupaloy.New(cupaloy.SnapshotSubdirectory("testdata"))
 
 	t.Run("Markdown", func(t *testing.T) {
 		tmpdir := t.TempDir()
 
-		err := p.GeneratePluginDocs(tmpdir, "markdown")
+		err := g.Generate(tmpdir, FormatMarkdown)
 		if err != nil {
 			t.Fatalf("unexpected error calling GeneratePluginDocs: %v", err)
 		}
@@ -146,7 +145,7 @@ func TestGeneratePluginDocs(t *testing.T) {
 	t.Run("JSON", func(t *testing.T) {
 		tmpdir := t.TempDir()
 
-		err := p.GeneratePluginDocs(tmpdir, "json")
+		err := g.Generate(tmpdir, FormatJSON)
 		if err != nil {
 			t.Fatalf("unexpected error calling GeneratePluginDocs: %v", err)
 		}

docs/generator.go

@@ -0,0 +1,62 @@
+package docs
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"path/filepath"
+
+	"github.com/cloudquery/plugin-sdk/v4/schema"
+)
+
+type jsonTable struct {
+	Name        string       `json:"name"`
+	Title       string       `json:"title"`
+	Description string       `json:"description"`
+	Columns     []jsonColumn `json:"columns"`
+	Relations   []jsonTable  `json:"relations"`
+}
+
+type jsonColumn struct {
+	Name             string `json:"name"`
+	Type             string `json:"type"`
+	IsPrimaryKey     bool   `json:"is_primary_key,omitempty"`
+	IsIncrementalKey bool   `json:"is_incremental_key,omitempty"`
+}
+
+func (g *Generator) renderTablesAsJSON(dir string) error {
+	jsonTables := g.jsonifyTables(g.tables)
+	buffer := &bytes.Buffer{}
+	m := json.NewEncoder(buffer)
+	m.SetIndent("", "  ")
+	m.SetEscapeHTML(false)
+	err := m.Encode(jsonTables)
+	if err != nil {
+		return err
+	}
+	outputPath := filepath.Join(dir, "__tables.json")
+	return os.WriteFile(outputPath, buffer.Bytes(), 0644)
+}
+
+func (g *Generator) jsonifyTables(tables schema.Tables) []jsonTable {
+	jsonTables := make([]jsonTable, len(tables))
+	for i, table := range tables {
+		jsonColumns := make([]jsonColumn, len(table.Columns))
+		for c, col := range table.Columns {
+			jsonColumns[c] = jsonColumn{
+				Name:             col.Name,
+				Type:             col.Type.String(),
+				IsPrimaryKey:     col.PrimaryKey,
+				IsIncrementalKey: col.IncrementalKey,
+			}
+		}
+		jsonTables[i] = jsonTable{
+			Name:        table.Name,
+			Title:       g.titleTransformer(table),
+			Description: table.Description,
+			Columns:     jsonColumns,
+			Relations:   g.jsonifyTables(table.Relations),
+		}
+	}
+	return jsonTables
+}

plugins/source/docs_test.go renamed to docs/generator_test.go

@@ -0,0 +1,94 @@
+package docs
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+	"text/template"
+
+	"github.com/cloudquery/plugin-sdk/v4/schema"
+)
+
+type templateData struct {
+	PluginName string
+	Tables     schema.Tables
+}
+
+func (g *Generator) renderTablesAsMarkdown(dir string) error {
+	for _, table := range g.tables {
+		if err := g.renderAllTables(dir, table); err != nil {
+			return err
+		}
+	}
+	t, err := template.New("all_tables.md.go.tpl").Funcs(template.FuncMap{
+		"indentToDepth": indentToDepth,
+	}).ParseFS(templatesFS, "templates/all_tables*.md.go.tpl")
+	if err != nil {
+		return fmt.Errorf("failed to parse template for README.md: %v", err)
+	}
+
+	var b bytes.Buffer
+	if err := t.Execute(&b, templateData{PluginName: g.pluginName, Tables: g.tables}); err != nil {
+		return fmt.Errorf("failed to execute template: %v", err)
+	}
+	content := formatMarkdown(b.String())
+	outputPath := filepath.Join(dir, "README.md")
+	f, err := os.Create(outputPath)
+	if err != nil {
+		return fmt.Errorf("failed to create file %v: %v", outputPath, err)
+	}
+	f.WriteString(content)
+	return nil
+}
+
+func (g *Generator) renderAllTables(dir string, t *schema.Table) error {
+	if err := g.renderTable(dir, t); err != nil {
+		return err
+	}
+	for _, r := range t.Relations {
+		if err := g.renderAllTables(dir, r); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (g *Generator) renderTable(dir string, table *schema.Table) error {
+	t := template.New("").Funcs(map[string]any{
+		"title": g.titleTransformer,
+	})
+	t, err := t.New("table.md.go.tpl").ParseFS(templatesFS, "templates/table.md.go.tpl")
+	if err != nil {
+		return fmt.Errorf("failed to parse template: %v", err)
+	}
+
+	outputPath := filepath.Join(dir, fmt.Sprintf("%s.md", table.Name))
+
+	var b bytes.Buffer
+	if err := t.Execute(&b, table); err != nil {
+		return fmt.Errorf("failed to execute template: %v", err)
+	}
+	content := formatMarkdown(b.String())
+	f, err := os.Create(outputPath)
+	if err != nil {
+		return fmt.Errorf("failed to create file %v: %v", outputPath, err)
+	}
+	f.WriteString(content)
+	return f.Close()
+}
+
+func formatMarkdown(s string) string {
+	s = reMatchNewlines.ReplaceAllString(s, "\n\n")
+	return reMatchHeaders.ReplaceAllString(s, `$1`+"\n\n")
+}
+
+func indentToDepth(table *schema.Table) string {
+	s := ""
+	t := table
+	for t.Parent != nil {
+		s += "  "
+		t = t.Parent
+	}
+	return s
+}