|
1 |
| -{% macro calc_batch_size(num_columns,max_batch_size) %} |
2 |
| - {# |
3 |
| - SQL Server allows for a max of 2100 parameters in a single statement. |
4 |
| - Check if the max_batch_size fits with the number of columns, otherwise |
5 |
| - reduce the batch size so it fits. |
6 |
| - #} |
7 |
| - {% if num_columns * max_batch_size < 2100 %} |
8 |
| - {% set batch_size = max_batch_size %} |
9 |
| - {% else %} |
10 |
| - {% set batch_size = (2100 / num_columns)|int %} |
11 |
| - {% endif %} |
12 |
| - |
13 |
| - {{ return(batch_size) }} |
14 |
| -{% endmacro %} |
15 |
| - |
16 | 1 | {% macro sqlserver__get_binding_char() %}
|
17 | 2 | {{ return('?') }}
|
18 | 3 | {% endmacro %}
|
|
21 | 6 | {{ return(400) }}
|
22 | 7 | {% endmacro %}
|
23 | 8 |
|
24 |
| -{% macro basic_load_csv_rows(model, batch_size, agate_table) %} |
| 9 | +{% macro calc_batch_size(num_columns) %} |
| 10 | + {# |
| 11 | + SQL Server allows for a max of 2100 parameters in a single statement. |
| 12 | + Check if the max_batch_size fits with the number of columns, otherwise |
| 13 | + reduce the batch size so it fits. |
| 14 | + #} |
| 15 | + {% set max_batch_size = get_batch_size() %} |
| 16 | + {% set calculated_batch = (2100 / num_columns)|int %} |
| 17 | + {% set batch_size = [max_batch_size, calculated_batch] | min %} |
| 18 | + |
| 19 | + {{ return(batch_size) }} |
| 20 | +{% endmacro %} |
25 | 21 |
|
| 22 | +{% macro sqlserver__load_csv_rows(model, agate_table) %} |
26 | 23 | {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
|
| 24 | + {% set batch_size = calc_batch_size(agate_table.column_names|length) %} |
27 | 25 | {% set bindings = [] %}
|
28 |
| - |
29 | 26 | {% set statements = [] %}
|
30 | 27 |
|
| 28 | + {{ log("Inserting batches of " ~ batch_size ~ " records") }} |
| 29 | + |
31 | 30 | {% for chunk in agate_table.rows | batch(batch_size) %}
|
32 | 31 | {% set bindings = [] %}
|
33 | 32 |
|
|
56 | 55 | {# Return SQL so we can render it out into the compiled files #}
|
57 | 56 | {{ return(statements[0]) }}
|
58 | 57 | {% endmacro %}
|
59 |
| - |
60 |
| -{% macro sqlserver__load_csv_rows(model, agate_table) %} |
61 |
| - {% set max_batch_size = get_batch_size() %} |
62 |
| - {% set cols_sql = agate_table.column_names %} |
63 |
| - {% set batch_size = calc_batch_size(cols_sql|length, max_batch_size) %} |
64 |
| - |
65 |
| - {{ return(basic_load_csv_rows(model, batch_size, agate_table) )}} |
66 |
| -{% endmacro %} |
0 commit comments