Skip to content

Commit 29e82db

Browse files
committed
bugfix to fix_seed_if_needed
1 parent 7667f8e commit 29e82db

File tree

2 files changed

+26
-15
lines changed

2 files changed

+26
-15
lines changed
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{% macro replace_empty_strings_with_nulls(table_name) %}
2+
{% set relation = ref(table_name) %}
3+
{% set columns = adapter.get_columns_in_relation(relation) %}
4+
5+
{% for col in columns %}
6+
{% set data_type = elementary.get_column_data_type(col) %}
7+
{% set normalized_data_type = elementary.normalize_data_type(data_type) %}
8+
9+
{% if normalized_data_type == "string" %}
10+
{% set update_query %}
11+
update {{ relation }}
12+
set {{ col["name"] }} = NULL
13+
where {{ col["name"] }} = ''
14+
{% endset %}
15+
{% do elementary.run_query(update_query) %}
16+
{% endif %}
17+
{% endfor %}
18+
{% endmacro %}

integration_tests/tests/dbt_project.py

Lines changed: 8 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -260,23 +260,16 @@ def seed(self, data: List[dict], table_name: str):
260260
with DbtDataSeeder(
261261
self.dbt_runner, self.project_dir_path, self.seeds_dir_path
262262
).seed(data, table_name):
263-
self._fix_seed_if_needed(data, table_name)
263+
self._fix_seed_if_needed(table_name)
264264

265-
def _fix_seed_if_needed(self, data: List[dict], table_name: str):
266-
# Ugly hack for bigquery + fusion - seems like nulls are seeded as empty strings
265+
def _fix_seed_if_needed(self, table_name: str):
266+
# Hack for BigQuery - seems like we get empty strings instead of nulls in seeds, so we
267+
# fix them here
267268
if self.runner_method == RunnerMethod.FUSION and self.target == "bigquery":
268-
for col_name in data[0].keys():
269-
self.run_query(
270-
"""
271-
UPDATE {{ ref('%(table_name)s') }}
272-
SET %(col_name)s = NULL
273-
WHERE %(col_name)s = ''
274-
"""
275-
% {
276-
"table_name": table_name,
277-
"col_name": col_name,
278-
}
279-
)
269+
self.dbt_runner.run_operation(
270+
"elementary_tests.replace_empty_strings_with_nulls",
271+
macro_args={"table_name": table_name},
272+
)
280273

281274
@contextmanager
282275
def seed_context(

0 commit comments

Comments
 (0)