use table definition cache (#12)

* use table definition cache

* test cache in test

* Update cmd/pgtalk-gen/main.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update cmd/pgtalk-gen/main.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update cmd/pgtalk-gen/main.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* update readme, changes, dependencies

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: emicklei

.gitignore

@@ -1,3 +1,4 @@
 .vscode
 cmd/pgtalk-gen/pgtalk-gen
 test/gen.sh
+test/pgtalk-tables-cache.json

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 All notable changes to this project will be documented in this file.
 
+## [v1.13.0] - 2025-08-19
+
+- add option `-cache` to store table definitions in JSON for generation without db server.
+
 ## [v1.12.1] - 2025-05-25
 
 - add Stringer to QuerySet and MutationSet, update deps

README.md

@@ -246,4 +246,13 @@ The configuration for this mapping is:
 		}
 	}
 
+## cache table definitions read from database
+
+If you do not want to add generated sources to your SCM (source code management system e.g. git) then build systems and other developers need to re-generated them.
+The `pgtalk-gen` uses table definitions from an active database server to generate Go table types.
+With the `-cache` option, this tool can store those definitions in a JSON file which then can be used instead and you add that file to your SCM.
+It is the responsibility of the developer to update it (by deleting it) upon each table definition change in the database.
+See the `test` folder for an example; just run all the tasks in the `Makefile`.
+
+
 (c) 2025, https://ernestmicklei.com. MIT License.

cmd/pgtalk-gen/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"log"
@@ -17,23 +18,52 @@ var (
 	oViews           = flag.Bool("views", false, "generated from views, default is false = use tables")
 	oVerbose         = flag.Bool("v", false, "use verbose logging")
 	oIncludePatterns = flag.String("include", ".*", "comma separated list of regexp for tables to include")
-	oExludePatterns  = flag.String("exclude", "", "comma separated list of regexp for tables to exclude")
+	oExcludePatterns = flag.String("exclude", "", "comma separated list of regexp for tables to exclude")
 	oMapping         = flag.String("mapping", "", "mapping file for undefined pg data types")
+	oCache           = flag.String("cache", "", "use cache file for table metadata if present")
 )
 
 func main() {
 	flag.Parse()
-	connectionString := os.Getenv("PGTALK_CONN")
-	if len(connectionString) == 0 {
-		fmt.Fprintf(os.Stderr, "Missing value of environment variable PGTALK_CONN\n")
-		os.Exit(1)
-	}
-
 	if err := applyConfiguredMappings(*oMapping); err != nil {
 		fmt.Fprintf(os.Stderr, "Unable to process custom mappings: %v\n", err)
 		os.Exit(1)
 	}
+	var tables []PgTable
+	if *oCache != "" {
+		if result, err := loadPgTablesFromCache(*oCache); err != nil {
+			fmt.Fprintf(os.Stderr, "unable to load cache file: %v\ncontinue using database connection", err)
+		} else {
+			tables = result
+		}
+	}
+	if len(tables) == 0 {
+		tables = fetchPgTables()
+		if *oCache != "" {
+			if err := savePgTablesToCache(*oCache, tables); err != nil {
+				fmt.Fprintf(os.Stderr, "unable to save cache file: %v\n", err)
+			}
+		}
+	}
+	filter := NewTableFilter(*oIncludePatterns, *oExcludePatterns)
+	for _, each := range tables {
+		if filter.Includes(each.Name) {
+			if *oDryrun {
+				log.Println("[-dry] would generate", each.Name)
+			} else {
+				generateFromTable(each, *oViews)
+			}
+		}
+	}
+}
 
+func fetchPgTables() []PgTable {
+	connectionString := os.Getenv("PGTALK_CONN")
+	if len(connectionString) == 0 {
+		fmt.Fprintf(os.Stderr, "Missing value of environment variable PGTALK_CONN\n")
+		os.Exit(1)
+	}
+	log.Println("fetching tables from database")
 	conn, err := pgx.Connect(context.Background(), connectionString)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Unable to connect to database: %v\n", err)
@@ -53,14 +83,30 @@ func main() {
 			log.Fatal(err)
 		}
 	}
-	filter := NewTableFilter(*oIncludePatterns, *oExludePatterns)
-	for _, each := range all {
-		if filter.Includes(each.Name) {
-			if *oDryrun {
-				log.Println("[DRYRUN] would generate", each.Name)
-			} else {
-				generateFromTable(each, *oViews)
-			}
-		}
+	return all
+}
+
+func loadPgTablesFromCache(cacheFile string) ([]PgTable, error) {
+	log.Printf("loading tables from cache: %s", cacheFile)
+	data, err := os.ReadFile(cacheFile)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read cache file: %w", err)
+	}
+	var tables []PgTable
+	if err := json.Unmarshal(data, &tables); err != nil {
+		return nil, fmt.Errorf("unable to unmarshal cache file: %w", err)
+	}
+	return tables, nil
+}
+
+func savePgTablesToCache(cacheFile string, tables []PgTable) error {
+	log.Printf("saving tables to cache: %s", cacheFile)
+	data, err := json.MarshalIndent(tables, "", "  ")
+	if err != nil {
+		return fmt.Errorf("unable to marshal tables to JSON: %w", err)
+	}
+	if err := os.WriteFile(cacheFile, data, 0644); err != nil {
+		return fmt.Errorf("unable to write cache file: %w", err)
 	}
+	return nil
 }

go.mod

@@ -13,6 +13,6 @@ require (
 require (
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
-	golang.org/x/crypto v0.40.0 // indirect
-	golang.org/x/text v0.27.0 // indirect
+	golang.org/x/crypto v0.41.0 // indirect
+	golang.org/x/text v0.28.0 // indirect
 )

go.sum

@@ -24,12 +24,12 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM=
-golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY=
+golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=
+golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=
 golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
 golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
-golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
-golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=
+golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
+golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

test/Makefile

@@ -1,6 +1,3 @@
-rmdb:
-	docker rm -f pgtalk-srv
-
 dbsrv:
 	docker run --name pgtalk-srv -e POSTGRES_PASSWORD=pgtalk  -p 5432:5432 -d postgres
 
@@ -9,8 +6,11 @@ db:
 
 gen:
 	cd ../cmd/pgtalk-gen && go install
-	PGTALK_CONN=postgres://postgres:pgtalk@localhost:5432/pgtalk-db pgtalk-gen -v -o . -include things,products,categories -mapping types/mapping.json
+	PGTALK_CONN=postgres://postgres:pgtalk@localhost:5432/pgtalk-db pgtalk-gen -v -o . -include things,products,categories -mapping types/mapping.json -cache pgtalk-tables-cache.json
 	# gofmt -w .
 
 test:
-	PGTALK_CONN=postgres://postgres:pgtalk@localhost:5432/pgtalk-db go test ${FLAGS}
+	PGTALK_CONN=postgres://postgres:pgtalk@localhost:5432/pgtalk-db go test ${FLAGS}
+
+rmdb:
+	docker rm -f pgtalk-srv