Skip to content

Commit 011315c

Browse files
authored
Fix: Resolve load failures for large files by running Snowflake PUT queries sequentially (#188)
1 parent 6c9dab8 commit 011315c

File tree

1 file changed

+4
-4
lines changed

1 file changed

+4
-4
lines changed

airbyte/_processors/sql/snowflake.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,10 @@ def _write_files_to_new_table(
6969
def path_str(path: Path) -> str:
7070
return str(path.absolute()).replace("\\", "\\\\")
7171

72-
put_files_statements = "\n".join(
73-
[f"PUT 'file://{path_str(file_path)}' {internal_sf_stage_name};" for file_path in files]
74-
)
75-
self._execute_sql(put_files_statements)
72+
for file_path in files:
73+
query = f"PUT 'file://{path_str(file_path)}' {internal_sf_stage_name};"
74+
self._execute_sql(query)
75+
7676
columns_list = [
7777
self._quote_identifier(c)
7878
for c in list(self._get_sql_column_definitions(stream_name).keys())

0 commit comments

Comments
 (0)