Skip to content

Commit 38ba450

Browse files
modular-magicianmodular-magician
authored
Fix bug where bigquery table couldn't add columns (#4606) (#3100)
Signed-off-by: Modular Magician <[email protected]>
1 parent 0df3171 commit 38ba450

File tree

3 files changed

+128
-8
lines changed

3 files changed

+128
-8
lines changed

‎.changelog/4606.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
```release-note:bug
2+
bigquery: fixed issue where you couldn't extend an existing `schema` with additional columns in `google_bigquery_table`
3+
```

‎google-beta/resource_bigquery_table.go‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,15 @@ func bigQueryTableSortArrayByName(array []interface{}) {
2121
})
2222
}
2323

24+
func bigQueryArrayToMapIndexedByName(array []interface{}) map[string]interface{} {
25+
out := map[string]interface{}{}
26+
for _, v := range array {
27+
name := v.(map[string]interface{})["name"].(string)
28+
out[name] = v
29+
}
30+
return out
31+
}
32+
2433
func bigQueryTablecheckNameExists(jsonList []interface{}) error {
2534
for _, m := range jsonList {
2635
if _, ok := m.(map[string]interface{})["name"]; !ok {
@@ -197,14 +206,18 @@ func resourceBigQueryTableSchemaIsChangeable(old, new interface{}) (bool, error)
197206
if err := bigQueryTablecheckNameExists(arrayOld); err != nil {
198207
return false, err
199208
}
200-
bigQueryTableSortArrayByName(arrayOld)
209+
mapOld := bigQueryArrayToMapIndexedByName(arrayOld)
201210
if err := bigQueryTablecheckNameExists(arrayNew); err != nil {
202211
return false, err
203212
}
204-
bigQueryTableSortArrayByName(arrayNew)
205-
for i := range arrayOld {
213+
mapNew := bigQueryArrayToMapIndexedByName(arrayNew)
214+
for key := range mapOld {
215+
// all old keys should be represented in the new config
216+
if _, ok := mapNew[key]; !ok {
217+
return false, nil
218+
}
206219
if isChangable, err :=
207-
resourceBigQueryTableSchemaIsChangeable(arrayOld[i], arrayNew[i]); err != nil || !isChangable {
220+
resourceBigQueryTableSchemaIsChangeable(mapOld[key], mapNew[key]); err != nil || !isChangable {
208221
return false, err
209222
}
210223
}
@@ -216,15 +229,13 @@ func resourceBigQueryTableSchemaIsChangeable(old, new interface{}) (bool, error)
216229
// if both aren't objects
217230
return false, nil
218231
}
219-
220232
var unionOfKeys map[string]bool = make(map[string]bool)
221233
for key := range objectOld {
222234
unionOfKeys[key] = true
223235
}
224236
for key := range objectNew {
225237
unionOfKeys[key] = true
226238
}
227-
228239
for key := range unionOfKeys {
229240
valOld := objectOld[key]
230241
valNew := objectNew[key]

‎google-beta/resource_bigquery_table_test.go‎

Lines changed: 108 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -509,6 +509,39 @@ func TestAccBigQueryDataTable_canReorderParameters(t *testing.T) {
509509
})
510510
}
511511

512+
func TestAccBigQueryDataTable_expandArray(t *testing.T) {
513+
t.Parallel()
514+
515+
datasetID := fmt.Sprintf("tf_test_%s", randString(t, 10))
516+
tableID := fmt.Sprintf("tf_test_%s", randString(t, 10))
517+
518+
vcrTest(t, resource.TestCase{
519+
PreCheck: func() { testAccPreCheck(t) },
520+
Providers: testAccProviders,
521+
CheckDestroy: testAccCheckBigQueryTableDestroyProducer(t),
522+
Steps: []resource.TestStep{
523+
{
524+
Config: testAccBigQueryTable_arrayInitial(datasetID, tableID),
525+
},
526+
{
527+
ResourceName: "google_bigquery_table.test",
528+
ImportState: true,
529+
ImportStateVerify: true,
530+
ImportStateVerifyIgnore: []string{"etag", "last_modified_time", "deletion_protection"},
531+
},
532+
{
533+
Config: testAccBigQueryTable_arrayExpanded(datasetID, tableID),
534+
},
535+
{
536+
ResourceName: "google_bigquery_table.test",
537+
ImportState: true,
538+
ImportStateVerify: true,
539+
ImportStateVerifyIgnore: []string{"etag", "last_modified_time", "deletion_protection"},
540+
},
541+
},
542+
})
543+
}
544+
512545
func TestUnitBigQueryDataTable_jsonEquivalency(t *testing.T) {
513546
t.Parallel()
514547

@@ -632,12 +665,12 @@ var testUnitBigQueryDataTableIsChangableTestCases = []testUnitBigQueryDataTableJ
632665
{
633666
name: "arraySizeIncreases",
634667
jsonOld: "[{\"name\": \"someValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }]",
635-
jsonNew: "[{\"name\": \"someValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }, {\"name\": \"someValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }]",
668+
jsonNew: "[{\"name\": \"someValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }, {\"name\": \"asomeValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }]",
636669
changeable: true,
637670
},
638671
{
639672
name: "arraySizeDecreases",
640-
jsonOld: "[{\"name\": \"someValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }, {\"name\": \"someValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }]",
673+
jsonOld: "[{\"name\": \"someValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }, {\"name\": \"asomeValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }]",
641674
jsonNew: "[{\"name\": \"someValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }]",
642675
changeable: false,
643676
},
@@ -1630,6 +1663,79 @@ resource "google_bigquery_table" "test" {
16301663
`, datasetID, tableID)
16311664
}
16321665

1666+
func testAccBigQueryTable_arrayInitial(datasetID, tableID string) string {
1667+
return fmt.Sprintf(`
1668+
resource "google_bigquery_dataset" "test" {
1669+
dataset_id = "%s"
1670+
}
1671+
1672+
resource "google_bigquery_table" "test" {
1673+
deletion_protection = false
1674+
table_id = "%s"
1675+
dataset_id = google_bigquery_dataset.test.dataset_id
1676+
1677+
friendly_name = "bigquerytest"
1678+
labels = {
1679+
"terrafrom_managed" = "true"
1680+
}
1681+
1682+
schema = jsonencode(
1683+
[
1684+
{
1685+
description = "Time snapshot was taken, in Epoch milliseconds. Same across all rows and all tables in the snapshot, and uniquely defines a particular snapshot."
1686+
name = "snapshot_timestamp"
1687+
mode = "NULLABLE"
1688+
type = "INTEGER"
1689+
},
1690+
{
1691+
description = "Timestamp of dataset creation"
1692+
name = "creation_time"
1693+
type = "TIMESTAMP"
1694+
},
1695+
])
1696+
}
1697+
`, datasetID, tableID)
1698+
}
1699+
1700+
func testAccBigQueryTable_arrayExpanded(datasetID, tableID string) string {
1701+
return fmt.Sprintf(`
1702+
resource "google_bigquery_dataset" "test" {
1703+
dataset_id = "%s"
1704+
}
1705+
1706+
resource "google_bigquery_table" "test" {
1707+
deletion_protection = false
1708+
table_id = "%s"
1709+
dataset_id = google_bigquery_dataset.test.dataset_id
1710+
1711+
friendly_name = "bigquerytest"
1712+
labels = {
1713+
"terrafrom_managed" = "true"
1714+
}
1715+
1716+
schema = jsonencode(
1717+
[
1718+
{
1719+
description = "Time snapshot was taken, in Epoch milliseconds. Same across all rows and all tables in the snapshot, and uniquely defines a particular snapshot."
1720+
name = "snapshot_timestamp"
1721+
mode = "NULLABLE"
1722+
type = "INTEGER"
1723+
},
1724+
{
1725+
description = "Timestamp of dataset creation"
1726+
name = "creation_time"
1727+
type = "TIMESTAMP"
1728+
},
1729+
{
1730+
description = "some new value"
1731+
name = "a_new_value"
1732+
type = "TIMESTAMP"
1733+
},
1734+
])
1735+
}
1736+
`, datasetID, tableID)
1737+
}
1738+
16331739
var TEST_CSV = `lifelock,LifeLock,,web,Tempe,AZ,1-May-07,6850000,USD,b
16341740
lifelock,LifeLock,,web,Tempe,AZ,1-Oct-06,6000000,USD,a
16351741
lifelock,LifeLock,,web,Tempe,AZ,1-Jan-08,25000000,USD,c

0 commit comments

Comments
 (0)