Skip to content

Commit 95229d6

Browse files
DVAlexHiggsDVAlexHiggs
committed
Merge remote-tracking branch 'private/release/0.11.4'
2 parents abe2094 + b6fde32 commit 95229d6

File tree

11 files changed

+542
-44
lines changed

11 files changed

+542
-44
lines changed

‎NOTICE‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
AutomateDV (f.k.a dbtvault)
2-
Copyright 2019-2024 Business Thinking Ltd (Trading as Datavault).
2+
Copyright 2019-2025 Business Thinking Ltd (Trading as Datavault).
33

44
Licensed under the Apache License, Version 2.0 (the "License");
55
you may not use this file except in compliance with the License.

‎README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
</div>
88
<div align="center">
99

10-
[![dbt Versions](https://img.shields.io/badge/compatible%20dbt%20versions-%3E=1.4%20%3C=1.8.x-orange?logo=dbt)](https://automate-dv.readthedocs.io/en/latest/versions/)
10+
[![dbt Versions](https://img.shields.io/badge/compatible%20dbt%20versions-%3E=1.9.x%20%3C=3.0.0-orange?logo=dbt)](https://automate-dv.readthedocs.io/en/latest/versions/)
1111

1212
</div>
1313

@@ -36,7 +36,7 @@ Learn quickly with our worked example:
3636

3737
## Supported platforms:
3838

39-
[Platform support matrix](https://automate-dv.readthedocs.io/en/latest/platform_support/
39+
[Platform support matrix](https://automate-dv.readthedocs.io/en/latest/platform_support/)
4040

4141
## Installation
4242

‎dbt_project.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name: automate_dv
2-
version: 0.11.3
3-
require-dbt-version: [ ">=1.0.0", "<2.0.0" ]
2+
version: 0.11.4
3+
require-dbt-version: [ ">=1.0.0", "<3.0.0" ]
44
config-version: 2
55

66
model-paths: [ "models" ]

‎macros/internal/metadata_processing/process_payload_column_excludes.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
{%- do return(src_payload) -%}
1111
{%- endif -%}
1212

13-
{%- set source_model_cols = adapter.get_columns_in_relation(ref(source_model)) -%}
13+
{%- set source_model_cols = automate_dv.wrap_get_columns_in_relation(ref(source_model)) -%}
1414
{%- set columns_in_metadata = automate_dv.expand_column_list(
1515
columns=[src_pk, src_hashdiff, src_cdk,
1616
src_payload, src_extra_columns,

‎macros/internal/metadata_processing/wrap_get_columns_in_relation.sql‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{% macro wrap_get_columns_in_relation(node) -%}
2+
{{ return(adapter.get_columns_in_relation(node)) }}
3+
{% endmacro %}

‎macros/tables/databricks/eff_sat.sql‎

Lines changed: 157 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,162 @@
55

66
{%- macro databricks__eff_sat(src_pk, src_dfk, src_sfk, src_extra_columns, src_start_date, src_end_date, src_eff, src_ldts, src_source, source_model) -%}
77

8-
{{- automate_dv.default__eff_sat(src_pk=src_pk, src_dfk=src_dfk, src_sfk=src_sfk,
9-
src_extra_columns=src_extra_columns,
10-
src_start_date=src_start_date, src_end_date=src_end_date,
11-
src_eff=src_eff, src_ldts=src_ldts, src_source=src_source,
12-
source_model=source_model) -}}
8+
{%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_dfk, src_sfk, src_extra_columns, src_start_date, src_end_date, src_eff, src_ldts, src_source]) -%}
9+
{%- set fk_cols = automate_dv.expand_column_list(columns=[src_dfk, src_sfk]) -%}
10+
{%- set dfk_cols = automate_dv.expand_column_list(columns=[src_dfk]) -%}
11+
{%- set is_auto_end_dating = automate_dv.config_meta_get('is_auto_end_dating', default=false) %}
12+
13+
{%- set max_datetime = automate_dv.max_datetime() %}
14+
15+
WITH source_data AS (
16+
SELECT {{ automate_dv.prefix(source_cols, 'a', alias_target='source') }}
17+
FROM {{ ref(source_model) }} AS a
18+
WHERE {{ automate_dv.multikey(src_dfk, prefix='a', condition='IS NOT NULL') }}
19+
AND {{ automate_dv.multikey(src_sfk, prefix='a', condition='IS NOT NULL') }}
20+
{%- if model.config.materialized == 'vault_insert_by_period' %}
21+
AND __PERIOD_FILTER__
22+
{%- elif model.config.materialized == 'vault_insert_by_rank' %}
23+
AND __RANK_FILTER__
24+
{%- endif %}
25+
),
26+
27+
{%- if automate_dv.is_any_incremental() %}
28+
29+
{# Selecting the most recent records for each link hashkey -#}
30+
latest_records AS (
31+
SELECT {{ automate_dv.alias_all(source_cols, 'b') }}
32+
FROM {{ this }} AS b
33+
QUALIFY ROW_NUMBER() OVER (
34+
PARTITION BY {{ automate_dv.prefix([src_pk], 'b') }}
35+
ORDER BY b.{{ src_ldts }} DESC
36+
) = 1
37+
),
38+
39+
{# Selecting the open records of the most recent records for each link hashkey -#}
40+
latest_open AS (
41+
SELECT {{ automate_dv.alias_all(source_cols, 'c') }}
42+
FROM latest_records AS c
43+
WHERE {{ automate_dv.cast_date(automate_dv.alias(src_end_date, 'c')) }} = {{ automate_dv.cast_date(automate_dv.cast_datetime(max_datetime, as_string=true)) }}
44+
),
45+
46+
{# Selecting the closed records of the most recent records for each link hashkey -#}
47+
latest_closed AS (
48+
SELECT {{ automate_dv.alias_all(source_cols, 'd') }}
49+
FROM latest_records AS d
50+
WHERE {{ automate_dv.cast_date(automate_dv.alias(src_end_date, 'd')) }} != {{ automate_dv.cast_date(automate_dv.cast_datetime(max_datetime, as_string=true)) }}
51+
),
52+
53+
{# Identifying the completely new link relationships to be opened in eff sat -#}
54+
new_open_records AS (
55+
SELECT DISTINCT
56+
{{ automate_dv.alias_all([src_pk, fk_cols], 'f') }},
57+
{%- if automate_dv.is_something(src_extra_columns) %}
58+
{{ automate_dv.prefix([src_extra_columns], 'f') }},
59+
{% endif -%}
60+
{%- if is_auto_end_dating %}
61+
f.{{ src_eff }} AS {{ src_start_date }},
62+
{%- else %}
63+
f.{{ src_start_date }} AS {{ src_start_date }},
64+
{%- endif %}
65+
f.{{ src_end_date }} AS {{ src_end_date }},
66+
f.{{ src_eff }} AS {{ src_eff }},
67+
f.{{ src_ldts }},
68+
f.{{ src_source }}
69+
FROM source_data AS f
70+
LEFT ANTI JOIN latest_records AS lr
71+
ON {{ automate_dv.multikey(src_pk, prefix=['f','lr'], condition='=') }}
72+
),
73+
74+
{# Identifying the currently closed link relationships to be reopened in eff sat -#}
75+
new_reopened_records AS (
76+
SELECT DISTINCT
77+
{{ automate_dv.alias_all([src_pk, fk_cols], 'lc') }},
78+
{%- if automate_dv.is_something(src_extra_columns) %}
79+
{{ automate_dv.prefix([src_extra_columns], 'g') }},
80+
{% endif -%}
81+
{%- if is_auto_end_dating %}
82+
g.{{ src_eff }} AS {{ src_start_date }},
83+
{%- else %}
84+
g.{{ src_start_date }} AS {{ src_start_date }},
85+
{%- endif %}
86+
g.{{ src_end_date }} AS {{ src_end_date }},
87+
g.{{ src_eff }} AS {{ src_eff }},
88+
g.{{ src_ldts }},
89+
g.{{ src_source }}
90+
FROM source_data AS g
91+
INNER JOIN latest_closed AS lc
92+
ON {{ automate_dv.multikey(src_pk, prefix=['g','lc'], condition='=') }}
93+
WHERE {{ automate_dv.cast_date(automate_dv.alias(src_end_date, 'g')) }} = {{ automate_dv.cast_date(automate_dv.cast_datetime(max_datetime, as_string=true)) }}
94+
),
95+
96+
{# Creating the closing records -#}
97+
{# Identifying the currently open relationships that need to be closed due to change in SFK(s) -#}
98+
99+
{%- if is_auto_end_dating %}
100+
101+
new_closed_records AS (
102+
SELECT DISTINCT
103+
{{ automate_dv.alias_all([src_pk, fk_cols], 'lo') }},
104+
{% if automate_dv.is_something(src_extra_columns) %}
105+
{{ automate_dv.prefix([src_extra_columns], 'h') }},
106+
{% endif -%}
107+
lo.{{ src_start_date }} AS {{ src_start_date }},
108+
h.{{ src_eff }} AS {{ src_end_date }},
109+
h.{{ src_eff }} AS {{ src_eff }},
110+
h.{{ src_ldts }},
111+
lo.{{ src_source }}
112+
FROM source_data AS h
113+
INNER JOIN latest_open AS lo
114+
ON {{ automate_dv.multikey(src_dfk, prefix=['lo', 'h'], condition='=') }}
115+
WHERE NOT EXISTS (SELECT 1 FROM source_data AS stg WHERE {{ automate_dv.multikey(src_pk, prefix=['stg','lo'], condition='=') }}
116+
AND {{ automate_dv.multikey(src_end_date, prefix=['stg', 'lo'], condition='=') }} )
117+
),
118+
119+
{#- else if (not) is_auto_end_dating -#}
120+
{% else %}
121+
122+
new_closed_records AS (
123+
SELECT DISTINCT
124+
{{ automate_dv.alias_all([src_pk, fk_cols], 'lo') }},
125+
{% if automate_dv.is_something(src_extra_columns) %}
126+
{{ automate_dv.prefix([src_extra_columns], 'h') }},
127+
{% endif -%}
128+
h.{{ src_start_date }} AS {{ src_start_date }},
129+
h.{{ src_end_date }} AS {{ src_end_date }},
130+
h.{{ src_eff }} AS {{ src_eff }},
131+
h.{{ src_ldts }},
132+
lo.{{ src_source }}
133+
FROM source_data AS h
134+
LEFT OUTER JOIN latest_open AS lo
135+
ON {{ automate_dv.multikey(src_pk, prefix=['lo', 'h'], condition='=') }}
136+
LEFT ANTI JOIN latest_closed AS lc
137+
ON {{ automate_dv.multikey(src_pk, prefix=['lc', 'h'], condition='=') }}
138+
WHERE {{ automate_dv.cast_date(automate_dv.alias(src_end_date, 'h')) }} != {{ automate_dv.cast_date(automate_dv.cast_datetime(max_datetime, as_string=true)) }}
139+
AND {{ automate_dv.multikey(src_pk, prefix='lo', condition='IS NOT NULL') }}
140+
),
141+
142+
{#- end if is_auto_end_dating -#}
143+
{%- endif %}
144+
145+
records_to_insert AS (
146+
SELECT * FROM new_open_records
147+
UNION
148+
SELECT * FROM new_reopened_records
149+
UNION
150+
SELECT * FROM new_closed_records
151+
)
152+
153+
{#- else if not automate_dv.is_any_incremental() -#}
154+
{%- else %}
155+
156+
records_to_insert AS (
157+
SELECT {{ automate_dv.alias_all(source_cols, 'i') }}
158+
FROM source_data AS i
159+
)
160+
161+
{#- end if not automate_dv.is_any_incremental() -#}
162+
{%- endif %}
163+
164+
SELECT * FROM records_to_insert
13165

14166
{%- endmacro -%}

‎macros/tables/databricks/hub.sql‎

Lines changed: 82 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,87 @@
55

66
{%- macro databricks__hub(src_pk, src_nk, src_extra_columns, src_ldts, src_source, source_model) -%}
77

8-
{{- automate_dv.default__hub(src_pk=src_pk,
9-
src_nk=src_nk,
10-
src_extra_columns=src_extra_columns,
11-
src_ldts=src_ldts,
12-
src_source=src_source,
13-
source_model=source_model) -}}
8+
{%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_nk, src_extra_columns, src_ldts, src_source]) -%}
9+
10+
{%- if model.config.materialized == 'vault_insert_by_rank' %}
11+
{%- set source_cols_with_rank = source_cols + [automate_dv.config_meta_get('rank_column')] -%}
12+
{%- endif %}
13+
14+
{{ 'WITH ' -}}
15+
16+
{%- set stage_count = source_model | length -%}
17+
18+
{%- set ns = namespace(last_cte= "") -%}
19+
20+
{%- for src in source_model -%}
21+
22+
{%- set source_number = loop.index | string -%}
23+
24+
row_rank_{{ source_number }} AS (
25+
{%- if model.config.materialized == 'vault_insert_by_rank' %}
26+
SELECT {{ automate_dv.prefix(source_cols_with_rank, 'rr') }}
27+
{%- else %}
28+
SELECT {{ automate_dv.prefix(source_cols, 'rr') }}
29+
{%- endif %}
30+
FROM {{ ref(src) }} AS rr
31+
WHERE {{ automate_dv.multikey(src_pk, prefix='rr', condition='IS NOT NULL') }}
32+
QUALIFY ROW_NUMBER() OVER(
33+
PARTITION BY {{ automate_dv.prefix([src_pk], 'rr') }}
34+
ORDER BY {{ automate_dv.prefix([src_ldts], 'rr') }}
35+
) = 1
36+
{%- set ns.last_cte = "row_rank_{}".format(source_number) %}
37+
),{{ "\n" if not loop.last }}
38+
{% endfor -%}
39+
{% if stage_count > 1 %}
40+
stage_union AS (
41+
{%- for src in source_model %}
42+
SELECT * FROM row_rank_{{ loop.index | string }}
43+
{%- if not loop.last %}
44+
UNION ALL
45+
{%- endif %}
46+
{%- endfor %}
47+
{%- set ns.last_cte = "stage_union" %}
48+
),
49+
{%- endif -%}
50+
51+
{%- if model.config.materialized == 'vault_insert_by_period' %}
52+
stage_mat_filter AS (
53+
SELECT *
54+
FROM {{ ns.last_cte }}
55+
WHERE __PERIOD_FILTER__
56+
{%- set ns.last_cte = "stage_mat_filter" %}
57+
),
58+
{%- elif model.config.materialized == 'vault_insert_by_rank' %}
59+
stage_mat_filter AS (
60+
SELECT *
61+
FROM {{ ns.last_cte }}
62+
WHERE __RANK_FILTER__
63+
{%- set ns.last_cte = "stage_mat_filter" %}
64+
),
65+
{%- endif -%}
66+
67+
{%- if stage_count > 1 %}
68+
69+
row_rank_union AS (
70+
SELECT ru.*
71+
FROM {{ ns.last_cte }} AS ru
72+
WHERE {{ automate_dv.multikey(src_pk, prefix='ru', condition='IS NOT NULL') }}
73+
QUALIFY ROW_NUMBER() OVER(
74+
PARTITION BY {{ automate_dv.prefix([src_pk], 'ru') }}
75+
ORDER BY {{ automate_dv.prefix([src_ldts], 'ru') }}, {{ automate_dv.prefix([src_source], 'ru') }} ASC
76+
) = 1
77+
{%- set ns.last_cte = "row_rank_union" %}
78+
),
79+
{% endif %}
80+
records_to_insert AS (
81+
SELECT {{ automate_dv.prefix(source_cols, 'a', alias_target='target') }}
82+
FROM {{ ns.last_cte }} AS a
83+
{%- if automate_dv.is_any_incremental() %}
84+
LEFT ANTI JOIN {{ this }} AS d
85+
ON {{ automate_dv.multikey(src_pk, prefix=['a','d'], condition='=') }}
86+
{%- endif %}
87+
)
88+
89+
SELECT * FROM records_to_insert
1490

1591
{%- endmacro -%}

0 commit comments

Comments
 (0)