fix: Better handling for Arrow type strings in docs (#896)

This fixes the rendering of some Arrow types in the docs, as well as in the JSON

Author: hermanschaaf

plugins/source/docs.go

@@ -11,7 +11,6 @@ import (
 	"sort"
 	"text/template"
 
-	"github.com/apache/arrow/go/v13/arrow"
 	"github.com/cloudquery/plugin-sdk/v3/caser"
 	"github.com/cloudquery/plugin-sdk/v3/schema"
 )
@@ -128,12 +127,16 @@ type jsonColumn struct {
 
 func (p *Plugin) renderTablesAsJSON(dir string, tables schema.Tables) error {
 	jsonTables := p.jsonifyTables(tables)
-	b, err := json.MarshalIndent(jsonTables, "", "  ")
+	buffer := &bytes.Buffer{}
+	m := json.NewEncoder(buffer)
+	m.SetIndent("", "  ")
+	m.SetEscapeHTML(false)
+	err := m.Encode(jsonTables)
 	if err != nil {
-		return fmt.Errorf("failed to marshal tables as json: %v", err)
+		return err
 	}
 	outputPath := filepath.Join(dir, "__tables.json")
-	return os.WriteFile(outputPath, b, 0644)
+	return os.WriteFile(outputPath, buffer.Bytes(), 0644)
 }
 
 func (p *Plugin) jsonifyTables(tables schema.Tables) []jsonTable {
@@ -200,8 +203,7 @@ func (p *Plugin) renderAllTables(t *schema.Table, dir string) error {
 
 func (p *Plugin) renderTable(table *schema.Table, dir string) error {
 	t := template.New("").Funcs(map[string]any{
-		"formatType": formatType,
-		"title":      p.titleTransformer,
+		"title": p.titleTransformer,
 	})
 	t, err := t.New("table.md.go.tpl").ParseFS(templatesFS, "templates/table.md.go.tpl")
 	if err != nil {
@@ -228,10 +230,6 @@ func formatMarkdown(s string) string {
 	return reMatchHeaders.ReplaceAllString(s, `$1`+"\n\n")
 }
 
-func formatType(v arrow.DataType) string {
-	return v.String()
-}
-
 func indentToDepth(table *schema.Table) string {
 	s := ""
 	t := table

plugins/source/docs_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/apache/arrow/go/v13/arrow"
 	"github.com/bradleyjkemp/cupaloy/v2"
 	"github.com/cloudquery/plugin-sdk/v3/schema"
+	"github.com/cloudquery/plugin-sdk/v3/types"
 	"github.com/stretchr/testify/require"
 )
 
@@ -32,6 +33,22 @@ var testTables = []*schema.Table{
 				Type:       arrow.PrimitiveTypes.Int64,
 				PrimaryKey: true,
 			},
+			{
+				Name: "json_col",
+				Type: types.ExtensionTypes.JSON,
+			},
+			{
+				Name: "list_col",
+				Type: arrow.ListOf(arrow.PrimitiveTypes.Int64),
+			},
+			{
+				Name: "map_col",
+				Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int64),
+			},
+			{
+				Name: "struct_col",
+				Type: arrow.StructOf(arrow.Field{Name: "string_field", Type: arrow.BinaryTypes.String}, arrow.Field{Name: "int_field", Type: arrow.PrimitiveTypes.Int64}),
+			},
 		},
 		Relations: []*schema.Table{
 			{

plugins/source/templates/table.md.go.tpl

@@ -40,5 +40,5 @@ The following tables depend on {{.Name}}:
 | Name          | Type          |
 | ------------- | ------------- |
 {{- range $column := $.Columns }}
-|{{$column.Name}}{{if $column.PrimaryKey}} (PK){{end}}{{if $column.IncrementalKey}} (Incremental Key){{end}}|{{$column.Type | formatType}}|
+|{{$column.Name}}{{if $column.PrimaryKey}} (PK){{end}}{{if $column.IncrementalKey}} (Incremental Key){{end}}|{{$column.Type}}|
 {{- end }}

plugins/source/testdata/TestGeneratePluginDocs-JSON-__tables.json

@@ -72,6 +72,22 @@
         "name": "id_col2",
         "type": "int64",
         "is_primary_key": true
+      },
+      {
+        "name": "json_col",
+        "type": "json"
+      },
+      {
+        "name": "list_col",
+        "type": "list<item: int64, nullable>"
+      },
+      {
+        "name": "map_col",
+        "type": "map<utf8, int64, items_nullable>"
+      },
+      {
+        "name": "struct_col",
+        "type": "struct<string_field: utf8, int_field: int64>"
       }
     ],
     "relations": [
@@ -195,3 +211,4 @@
     ]
   }
 ]
+

plugins/source/testdata/TestGeneratePluginDocs-Markdown-test_table.md

@@ -23,3 +23,7 @@ The following tables depend on test_table:
 |int_col|int64|
 |id_col (PK)|int64|
 |id_col2 (PK)|int64|
+|json_col|json|
+|list_col|list<item: int64, nullable>|
+|map_col|map<utf8, int64, items_nullable>|
+|struct_col|struct<string_field: utf8, int_field: int64>|

types/inet.go

@@ -207,6 +207,10 @@ func (*InetType) ExtensionName() string {
 	return "inet"
 }
 
+func (*InetType) String() string {
+	return "inet"
+}
+
 // Serialize returns "inet-serialized" for testing proper metadata passing
 func (*InetType) Serialize() string {
 	return "inet-serialized"

types/json.go

@@ -209,8 +209,8 @@ func (*JSONType) ExtensionName() string {
 	return "json"
 }
 
-func (e *JSONType) String() string {
-	return fmt.Sprintf("extension_type<storage=%s>", e.Storage)
+func (*JSONType) String() string {
+	return "json"
 }
 
 func (e *JSONType) MarshalJSON() ([]byte, error) {