fix(package): Normalize tables when writing tables.json (#1227)

We might want to refactor this to no (ab)use the MemDB plugin but I think this is ok for now. Also we should call `FlattenTables` otherwise we only output the top level ones

---

Author: erezrokah

internal/memdb/memdb.go

@@ -43,7 +43,9 @@ func GetNewClient(options ...Option) plugin.NewClientFunc {
 	c := &client{
 		memoryDB:     make(map[string][]arrow.Record),
 		memoryDBLock: sync.RWMutex{},
-		tables:       make(map[string]*schema.Table),
+		tables: map[string]*schema.Table{
+			"table1": {Name: "table1", Relations: schema.Tables{{Name: "table2"}}},
+		},
 	}
 	for _, opt := range options {
 		opt(c)

serve/package.go

@@ -46,18 +46,50 @@ type TargetBuild struct {
 	Checksum string `json:"checksum"`
 }
 
+// This is the structure the CLI publish command expects
+type pluginTable struct {
+	Description   string    `json:"description,omitempty"`
+	IsIncremental bool      `json:"is_incremental,omitempty"`
+	Name          string    `json:"name,omitempty"`
+	Parent        *string   `json:"parent,omitempty"`
+	Relations     *[]string `json:"relations,omitempty"`
+	Title         string    `json:"title,omitempty"`
+}
+
 func (s *PluginServe) writeTablesJSON(ctx context.Context, dir string) error {
 	tables, err := s.plugin.Tables(ctx, plugin.TableOptions{
 		Tables: []string{"*"},
 	})
 	if err != nil {
 		return err
 	}
+	flattenedTables := tables.FlattenTables()
+	tablesToEncode := make([]pluginTable, 0, len(flattenedTables))
+	for _, t := range flattenedTables {
+		table := tables.Get(t.Name)
+		var parent *string
+		if table.Parent != nil {
+			parent = &table.Parent.Name
+		}
+		var relations *[]string
+		if table.Relations != nil {
+			names := table.Relations.TableNames()
+			relations = &names
+		}
+		tablesToEncode = append(tablesToEncode, pluginTable{
+			Description:   table.Description,
+			IsIncremental: table.IsIncremental,
+			Name:          table.Name,
+			Parent:        parent,
+			Relations:     relations,
+			Title:         table.Title,
+		})
+	}
 	buffer := &bytes.Buffer{}
 	m := json.NewEncoder(buffer)
-	m.SetIndent("", "  ")
+	m.SetIndent("", "")
 	m.SetEscapeHTML(false)
-	err = m.Encode(tables)
+	err = m.Encode(tablesToEncode)
 	if err != nil {
 		return err
 	}

serve/package_test.go

@@ -97,6 +97,7 @@ with multiple lines and **markdown**`
 				"overview.md",
 			}
 			checkDocs(t, filepath.Join(distDir, "docs"), expectDocs)
+			checkTables(t, distDir)
 		})
 	}
 }
@@ -125,6 +126,18 @@ func checkDocs(t *testing.T, dir string, expect []string) {
 	}
 }
 
+func checkTables(t *testing.T, distDir string) {
+	content, err := os.ReadFile(filepath.Join(distDir, "tables.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	tablesString := string(content)
+
+	if diff := cmp.Diff(tablesString, "[{\"name\":\"table1\",\"relations\":[\"table2\"]},{\"name\":\"table2\"}]\n"); diff != "" {
+		t.Fatalf("unexpected files in docs directory (-want +got):\n%s", diff)
+	}
+}
+
 func checkPackageJSONContents(t *testing.T, filename string, expect PackageJSON) {
 	f, err := os.Open(filename)
 	if err != nil {

serve/plugin_test.go

@@ -80,7 +80,7 @@ func TestPluginServe(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if len(tables) != 0 {
+	if len(tables) != 2 {
 		t.Fatalf("Expected 0 tables but got %d", len(tables))
 	}
 	testTable := schema.Table{