Skip to content

Commit 529532d

Browse files
erezrokaherezrokah
authored
feat: Add better changes summary helper (#2240)
#### Summary Part of https://linear.app/cloudquery/issue/ENG-1828/feat-improved-forced-migration-needed-error-message. Open for feedback, I tried to keep only the most interesting bit of each change. There's another improvement to be made and that's to pass only the breaking changes to the summary function (at the moment plugins print all changes which adds additional noise) ## Before ``` Loading spec(s) from examples/gcp_to_postgres.yml Starting migration for: gcp (cloudquery/[email protected]) -> [postgresql (cloudquery/[email protected])] Error: failed to migrate v3 source gcp: rpc error: code = Internal desc = write failed: tables gcp_compute_routes,gcp_artifactregistry_docker_images with changes [[column: next_hop_inter_region_cost, type: add, current: next_hop_inter_region_cost:int64 column: next_hop_med, type: add, current: next_hop_med:int64 column: next_hop_origin, type: add, current: next_hop_origin:utf8 column: tags, type: update, current: tags:json, previous: tags:list<item: utf8, nullable>] [column: tags, type: update, current: tags:json, previous: tags:list<item: utf8, nullable>]] require migration. Migrate manually or consider using 'migrate_mode: forced' ``` ## After ``` Loading spec(s) from examples/gcp_to_postgres.yml Starting migration for: gcp (cloudquery/[email protected]) -> [postgresql (grpc@localhost:8888)] Error: failed to migrate v3 source gcp: rpc error: code = Internal desc = write failed: Can't migrate tables automatically, migrate manually or consider using 'migrate_mode: forced'. Non auto migratable tables changes: gcp_artifactregistry_docker_images: - Type changed from "list<item: utf8, nullable>" to "json" for column "tags" gcp_compute_routes: - Column "next_hop_inter_region_cost" added with type "int64" - Column "next_hop_med" added with type "int64" - Column "next_hop_origin" added with type "utf8" - Type changed from "list<item: utf8, nullable>" to "json" for column "tags" ``` ---
1 parent eeed6a6 commit 529532d

File tree

1 file changed

+96
-0
lines changed

1 file changed

+96
-0
lines changed

‎schema/table.go‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,11 @@ import (
77
"fmt"
88
"regexp"
99
"slices"
10+
"strings"
1011

1112
"github.com/apache/arrow-go/v18/arrow"
1213
"github.com/cloudquery/plugin-sdk/v4/glob"
14+
"github.com/samber/lo"
1315
"github.com/thoas/go-funk"
1416
)
1517

@@ -258,6 +260,100 @@ func (t TableColumnChange) String() string {
258260
}
259261
}
260262

263+
func getColumnChangeSummary(change TableColumnChange) string {
264+
switch change.Type {
265+
case TableColumnChangeTypeAdd:
266+
if change.Current.PrimaryKey {
267+
return fmt.Sprintf("Column %q added with type %q and primary key constraint", change.ColumnName, change.Current.Type)
268+
}
269+
if change.Current.Unique {
270+
return fmt.Sprintf("Column %q added with type %q and unique constraint", change.ColumnName, change.Current.Type)
271+
}
272+
if change.Current.NotNull {
273+
return fmt.Sprintf("Column %q added with type %q and not null constraint", change.ColumnName, change.Current.Type)
274+
}
275+
if change.Current.IncrementalKey {
276+
return fmt.Sprintf("Column %q added with type %q and as an incremental key", change.ColumnName, change.Current.Type)
277+
}
278+
return fmt.Sprintf("Column %q added with type %q", change.ColumnName, change.Current.Type)
279+
case TableColumnChangeTypeUpdate:
280+
if change.Previous.Type != change.Current.Type {
281+
return fmt.Sprintf("Type changed from %q to %q for column %q", change.Previous.Type, change.Current.Type, change.ColumnName)
282+
}
283+
if !change.Previous.PrimaryKey && change.Current.PrimaryKey {
284+
return fmt.Sprintf("Primary key constraint added to column %q", change.ColumnName)
285+
}
286+
if change.Previous.PrimaryKey && !change.Current.PrimaryKey {
287+
return fmt.Sprintf("Primary key constraint removed from column %q", change.ColumnName)
288+
}
289+
if !change.Previous.Unique && change.Current.Unique {
290+
return fmt.Sprintf("Unique constraint added to column %q", change.ColumnName)
291+
}
292+
if change.Previous.Unique && !change.Current.Unique {
293+
return fmt.Sprintf("Unique constraint removed from column %q", change.ColumnName)
294+
}
295+
if !change.Previous.NotNull && change.Current.NotNull {
296+
return fmt.Sprintf("Not null constraint added to column %q", change.ColumnName)
297+
}
298+
if change.Previous.NotNull && !change.Current.NotNull {
299+
return fmt.Sprintf("Not null constraint removed from column %q", change.ColumnName)
300+
}
301+
if !change.Previous.IncrementalKey && change.Current.IncrementalKey {
302+
return fmt.Sprintf("Column %q added as an incremental key", change.ColumnName)
303+
}
304+
if change.Previous.IncrementalKey && !change.Current.IncrementalKey {
305+
return fmt.Sprintf("Column %q removed as an incremental key", change.ColumnName)
306+
}
307+
if !change.Previous.PrimaryKeyComponent && change.Current.PrimaryKeyComponent {
308+
return fmt.Sprintf("Primary key component condition added to column %q", change.ColumnName)
309+
}
310+
if change.Previous.PrimaryKeyComponent && !change.Current.PrimaryKeyComponent {
311+
return fmt.Sprintf("Primary key component condition removed from column %q", change.ColumnName)
312+
}
313+
return fmt.Sprintf("Column %q updated. Previous: %s, Current: %s", change.ColumnName, change.Previous, change.Current)
314+
case TableColumnChangeTypeRemove:
315+
if change.Previous.PrimaryKey {
316+
return fmt.Sprintf("Primary key column %q removed", change.ColumnName)
317+
}
318+
if change.Previous.Unique {
319+
return fmt.Sprintf("Unique column %q removed", change.ColumnName)
320+
}
321+
if change.Previous.NotNull {
322+
return fmt.Sprintf("Not null column %q removed", change.ColumnName)
323+
}
324+
if change.Previous.IncrementalKey {
325+
return fmt.Sprintf("Incremental key column %q removed", change.ColumnName)
326+
}
327+
return fmt.Sprintf("Column %q with type %q removed", change.ColumnName, change.Previous.Type)
328+
case TableColumnChangeTypeRemoveUniqueConstraint:
329+
return fmt.Sprintf("Unique constraint removed from column %q", change.ColumnName)
330+
case TableColumnChangeTypeMoveToCQOnly:
331+
return fmt.Sprintf("Primary key columns removed and replaced with a single column %q with type %q", change.ColumnName, change.Current.Type)
332+
default:
333+
return fmt.Sprintf("column: %s, type: %s, current: %s, previous: %s", change.ColumnName, change.Type, change.Current, change.Previous)
334+
}
335+
}
336+
337+
func GetChangesSummary(tablesChanges map[string][]TableColumnChange) string {
338+
tables := lo.Keys(tablesChanges)
339+
slices.Sort(tables)
340+
summary := strings.Builder{}
341+
for i, table := range tables {
342+
summary.WriteString(fmt.Sprintf("%s:\n", table))
343+
changes := tablesChanges[table]
344+
changesString := lo.Map(changes, func(change TableColumnChange, _ int) string {
345+
return fmt.Sprintf(" - %s", getColumnChangeSummary(change))
346+
})
347+
slices.Sort(changesString)
348+
summary.WriteString(strings.Join(changesString, "\n"))
349+
if i < len(tables)-1 {
350+
summary.WriteString("\n\n")
351+
}
352+
}
353+
354+
return summary.String()
355+
}
356+
261357
func (tt Tables) FilterDfsFunc(include, exclude func(*Table) bool, skipDependentTables bool) Tables {
262358
filteredTables := make(Tables, 0, len(tt))
263359
for _, t := range tt {

0 commit comments

Comments
 (0)