Skip to content

Commit 1320d32

Browse files
erezrokaherezrokah
authored
fix: Fail on empty tables (#796)
 Follow up to #790 ---
1 parent a61f34e commit 1320d32

File tree

11 files changed

+47
-14
lines changed

11 files changed

+47
-14
lines changed

‎specs/source.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,10 @@ func (s *Source) Validate() error {
113113
return fmt.Errorf(msg)
114114
}
115115

116+
if len(s.Tables) == 0 {
117+
return fmt.Errorf("tables configuration is required. Hint: set the tables you want to sync by adding `tables: [...]` or use `cloudquery tables` to list available tables")
118+
}
119+
116120
if s.Registry == RegistryGithub {
117121
if s.Version == "" {
118122
return fmt.Errorf("version is required")

‎specs/source_test.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ spec:
9393
spec:
9494
name: test
9595
path: cloudquery/test
96+
tables: ["test"]
9697
`,
9798
"version is required",
9899
nil,
@@ -104,17 +105,30 @@ spec:
104105
name: test
105106
path: cloudquery/test
106107
version: 1.1.0
108+
tables: ["test"]
107109
`,
108110
"version must start with v",
109111
nil,
110112
},
113+
{
114+
"tables_required",
115+
`kind: source
116+
spec:
117+
name: test
118+
path: cloudquery/test
119+
version: v1.1.0
120+
`,
121+
"tables configuration is required. Hint: set the tables you want to sync by adding `tables: [...]` or use `cloudquery tables` to list available tables",
122+
nil,
123+
},
111124
{
112125
"destination_required",
113126
`kind: source
114127
spec:
115128
name: test
116129
path: cloudquery/test
117130
version: v1.1.0
131+
tables: ["test"]
118132
`,
119133
"at least one destination is required",
120134
nil,
@@ -128,6 +142,7 @@ spec:
128142
version: v1.1.0
129143
destinations: ["test"]
130144
scheduler: round-robin
145+
tables: ["test"]
131146
`,
132147
"",
133148
&Source{
@@ -138,6 +153,7 @@ spec:
138153
Version: "v1.1.0",
139154
Destinations: []string{"test"},
140155
Scheduler: SchedulerRoundRobin,
156+
Tables: []string{"test"},
141157
},
142158
},
143159
{
@@ -148,6 +164,7 @@ spec:
148164
path: cloudquery/test
149165
version: v1.1.0
150166
destinations: ["test"]
167+
tables: ["test"]
151168
`,
152169
"",
153170
&Source{
@@ -158,6 +175,7 @@ spec:
158175
Version: "v1.1.0",
159176
Destinations: []string{"test"},
160177
Scheduler: SchedulerDFS,
178+
Tables: []string{"test"},
161179
},
162180
},
163181
}

‎specs/spec_reader_test.go‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ var specLoaderTestCases = []specLoaderTestCase{
3131
return ""
3232
},
3333
sources: []*Source{
34-
{Name: "gcp", Path: "cloudquery/gcp", Version: "v1.0.0", Registry: RegistryLocal, Destinations: []string{"postgresqlv2"}, TableConcurrency: 10},
35-
{Name: "aws", Path: "cloudquery/aws", Version: "v1.0.0", Registry: RegistryLocal, Destinations: []string{"postgresql"}, TableConcurrency: 10},
34+
{Name: "gcp", Path: "cloudquery/gcp", Version: "v1.0.0", Registry: RegistryLocal, Destinations: []string{"postgresqlv2"}, TableConcurrency: 10, Tables: []string{"test"}},
35+
{Name: "aws", Path: "cloudquery/aws", Version: "v1.0.0", Registry: RegistryLocal, Destinations: []string{"postgresql"}, TableConcurrency: 10, Tables: []string{"test"}},
3636
},
3737
destinations: []*Destination{
3838
{Name: "postgresqlv2", Path: "cloudquery/postgresql", Version: "v1.0.0", Registry: RegistryGrpc, WriteMode: WriteModeOverwrite, Spec: map[string]any{"credentials": "mytestcreds"}},
@@ -46,8 +46,8 @@ var specLoaderTestCases = []specLoaderTestCase{
4646
return ""
4747
},
4848
sources: []*Source{
49-
{Name: "gcp", Path: "cloudquery/gcp", Version: "v1.0.0", Registry: RegistryLocal, Destinations: []string{"postgresqlv2"}, TableConcurrency: 10},
50-
{Name: "aws", Path: "cloudquery/aws", Version: "v1.0.0", Registry: RegistryLocal, Destinations: []string{"postgresql"}, TableConcurrency: 10},
49+
{Name: "gcp", Path: "cloudquery/gcp", Version: "v1.0.0", Registry: RegistryLocal, Destinations: []string{"postgresqlv2"}, TableConcurrency: 10, Tables: []string{"test"}},
50+
{Name: "aws", Path: "cloudquery/aws", Version: "v1.0.0", Registry: RegistryLocal, Destinations: []string{"postgresql"}, TableConcurrency: 10, Tables: []string{"test"}},
5151
},
5252
destinations: []*Destination{
5353
{Name: "postgresqlv2", Path: "cloudquery/postgresql", Version: "v1.0.0", Registry: RegistryGrpc, WriteMode: WriteModeOverwrite, Spec: map[string]any{"credentials": "mytestcreds"}},
@@ -92,8 +92,8 @@ var specLoaderTestCases = []specLoaderTestCase{
9292
return ""
9393
},
9494
sources: []*Source{
95-
{Name: "aws", Path: "cloudquery/aws", Version: "v4.6.1", Registry: RegistryGithub, Destinations: []string{"postgresql"}},
96-
{Name: "azure", Path: "cloudquery/azure", Version: "v1.3.3", Registry: RegistryGithub, Destinations: []string{"postgresql"}},
95+
{Name: "aws", Path: "cloudquery/aws", Version: "v4.6.1", Registry: RegistryGithub, Destinations: []string{"postgresql"}, Tables: []string{"test"}},
96+
{Name: "azure", Path: "cloudquery/azure", Version: "v1.3.3", Registry: RegistryGithub, Destinations: []string{"postgresql"}, Tables: []string{"test"}},
9797
},
9898
destinations: []*Destination{
9999
{Name: "postgresql", Path: "cloudquery/postgresql", Version: "v1.6.3", Registry: RegistryGithub, Spec: map[string]any{"connection_string": "postgresql://postgres:pass@localhost:5432/postgres"}},
@@ -106,8 +106,8 @@ var specLoaderTestCases = []specLoaderTestCase{
106106
return ""
107107
},
108108
sources: []*Source{
109-
{Name: "aws", Path: "cloudquery/aws", Version: "v1", Registry: RegistryGithub, Destinations: []string{"postgresql"}},
110-
{Name: "azure", Path: "cloudquery/azure", Version: "v1.3.3", Registry: RegistryGithub, Destinations: []string{"postgresql", "postgresql"}},
109+
{Name: "aws", Path: "cloudquery/aws", Version: "v1", Registry: RegistryGithub, Destinations: []string{"postgresql"}, Tables: []string{"test"}},
110+
{Name: "azure", Path: "cloudquery/azure", Version: "v1.3.3", Registry: RegistryGithub, Destinations: []string{"postgresql", "postgresql"}, Tables: []string{"test"}},
111111
},
112112
destinations: []*Destination{
113113
{Name: "postgresql", Path: "cloudquery/postgresql", Version: "v1.6.3", Registry: RegistryGithub, Spec: map[string]any{"connection_string": "postgresql://localhost:5432/cloudquery?sslmode=disable", "version": "#v1"}},
@@ -125,8 +125,8 @@ var specLoaderTestCases = []specLoaderTestCase{
125125
return "failed to expand environment variable in file testdata/env_variables.yml (section 3): env variable CONNECTION_STRING not found"
126126
},
127127
sources: []*Source{
128-
{Name: "aws", Path: "cloudquery/aws", Version: "v1", Registry: RegistryGithub, Destinations: []string{"postgresql"}},
129-
{Name: "azure", Path: "cloudquery/azure", Version: "v1.3.3", Registry: RegistryGithub, Destinations: []string{"postgresql", "postgresql"}},
128+
{Name: "aws", Path: "cloudquery/aws", Version: "v1", Registry: RegistryGithub, Destinations: []string{"postgresql"}, Tables: []string{"test"}},
129+
{Name: "azure", Path: "cloudquery/azure", Version: "v1.3.3", Registry: RegistryGithub, Destinations: []string{"postgresql", "postgresql"}, Tables: []string{"test"}},
130130
},
131131
destinations: []*Destination{
132132
{Name: "postgresql", Path: "cloudquery/postgresql", Version: "v1.6.3", Registry: RegistryGithub, Spec: map[string]any{}},
@@ -143,7 +143,7 @@ var specLoaderTestCases = []specLoaderTestCase{
143143
return ""
144144
},
145145
sources: []*Source{
146-
{Name: "test", Path: "cloudquery/test", Version: "v1", Registry: RegistryGithub, Destinations: []string{"postgresql"}},
146+
{Name: "test", Path: "cloudquery/test", Version: "v1", Registry: RegistryGithub, Destinations: []string{"postgresql"}, Tables: []string{"test"}},
147147
},
148148
destinations: []*Destination{
149149
{Name: "postgresql", Path: "cloudquery/postgresql", Version: "v1", Registry: RegistryGithub, Spec: map[string]any{"custom_version": "#v1"}},
@@ -159,7 +159,7 @@ var specLoaderTestCases = []specLoaderTestCase{
159159
return "failed to expand environment variable in file testdata/env_variable_in_string.yml (section 2): env variable VERSION not found"
160160
},
161161
sources: []*Source{
162-
{Name: "test", Path: "cloudquery/test", Version: "v1", Registry: RegistryGithub, Destinations: []string{"postgresql"}},
162+
{Name: "test", Path: "cloudquery/test", Version: "v1", Registry: RegistryGithub, Destinations: []string{"postgresql"}, Tables: []string{"test"}},
163163
},
164164
destinations: []*Destination{
165165
{Name: "postgresql", Path: "cloudquery/postgresql", Version: "v1", Registry: RegistryGithub, Spec: map[string]any{}},
@@ -173,8 +173,8 @@ var specLoaderTestCases = []specLoaderTestCase{
173173
return ""
174174
},
175175
sources: []*Source{
176-
{Name: "0123456789", Path: "cloudquery/aws", Version: "v1", Registry: RegistryGithub, Destinations: []string{"0987654321"}},
177-
{Name: "012345", Path: "cloudquery/aws", Version: "v1", Registry: RegistryGithub, Destinations: []string{"0987654321"}},
176+
{Name: "0123456789", Path: "cloudquery/aws", Version: "v1", Registry: RegistryGithub, Destinations: []string{"0987654321"}, Tables: []string{"test"}},
177+
{Name: "012345", Path: "cloudquery/aws", Version: "v1", Registry: RegistryGithub, Destinations: []string{"0987654321"}, Tables: []string{"test"}},
178178
},
179179
destinations: []*Destination{
180180
{Name: "0987654321", Path: "cloudquery/postgresql", Version: "v1", Registry: RegistryGithub, Spec: map[string]any{"connection_string": "postgresql://localhost:5432/cloudquery?sslmode=disable"}},

‎specs/testdata/dir/aws.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ spec:
66
table_concurrency: 10
77
registry: local
88
destinations: [postgresql]
9+
tables: [test]

‎specs/testdata/dir_yaml/aws.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ spec:
66
table_concurrency: 10
77
registry: local
88
destinations: [postgresql]
9+
tables: [test]

‎specs/testdata/env_variable_in_string.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ spec:
44
path: cloudquery/test
55
version: "v1"
66
destinations: [postgresql]
7+
tables: [test]
78
---
89
kind: destination
910
spec:

‎specs/testdata/env_variables.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ spec:
44
version: ${VERSION}
55
destinations: [postgresql]
66
path: cloudquery/aws
7+
tables: [test]
78
---
89
# comment with ${ENV_VARIABLE} that should not be expanded
910
kind: source
@@ -12,6 +13,7 @@ spec:
1213
version: v1.3.3
1314
destinations: [${DESTINATIONS},postgresql]
1415
path: cloudquery/azure
16+
tables: [test]
1517
---
1618
kind: destination
1719
spec:

‎specs/testdata/gcp.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ spec:
66
table_concurrency: 10
77
registry: local
88
destinations: [postgresqlv2]
9+
tables: [test]
910
---
1011
kind: destination
1112
spec:

‎specs/testdata/gcpv2.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@ spec:
55
version: v1.0.1
66
table_concurrency: 10
77
destinations: [postgresqlv2]
8+
tables: [test]

‎specs/testdata/multiple_sources.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,15 @@ spec:
44
version: v4.6.1
55
destinations: [postgresql]
66
path: cloudquery/aws
7+
tables: [test]
78
---
89
kind: source
910
spec:
1011
name: azure
1112
version: v1.3.3
1213
destinations: [postgresql]
1314
path: cloudquery/azure
15+
tables: [test]
1416
---
1517
kind: destination
1618
spec:

0 commit comments

Comments
 (0)