Skip to content
Open
Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
004a784
Begin templating volatility in global scalar function macro
QMalcolm Oct 24, 2025
2561a20
Begin specifying minimum dbt-common version in dbt-tests-adapter to e…
QMalcolm Oct 29, 2025
2e2e2ac
Add basic test cases to `dbt-tests-adapter` for different volatility …
QMalcolm Oct 29, 2025
f9b1ae0
Test dbt-postgres support of volatility config
QMalcolm Oct 29, 2025
97f2784
Update snowflake scalar function macro to support volatility
QMalcolm Oct 29, 2025
fe15244
Begin testing snowflake udf volatility support
QMalcolm Oct 29, 2025
4912fc0
Support volatility setting in dbt-redshift sql functions
QMalcolm Oct 29, 2025
43bd82a
Begin testing volatility setting of sql functions in dbt-redshift
QMalcolm Oct 29, 2025
52aeb96
Add changie docs
QMalcolm Oct 29, 2025
644d71d
Test that bigquery does nothing with volatility
QMalcolm Oct 29, 2025
e8c1ad4
Update dbt-snowflake to issue warning when encountering `stable` vola…
QMalcolm Oct 30, 2025
c897e65
Update dbt-bigquery to raise warning when any volatility is present f…
QMalcolm Oct 30, 2025
8869308
Update test files used in bigquery volatility function tests
QMalcolm Oct 30, 2025
88bdd67
Update files used for redshift function volatility tests
QMalcolm Oct 30, 2025
9125f89
Ensure redshift volatility is being templated instead of hard coded
QMalcolm Oct 30, 2025
612f20c
Proper checking of `none` for bigquery volatility
QMalcolm Oct 30, 2025
0e628bf
Future proof volatility macros to catch any future unknown volatilities
QMalcolm Oct 30, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Add default macros for interpolating the volatility of sql UDFs
time: 2025-10-29T16:42:30.989678-05:00
custom:
Author: QMalcolm
Issue: "1345"
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
{% macro default__scalar_function_create_replace_signature_sql(target_relation) %}
CREATE OR REPLACE FUNCTION {{ target_relation.render() }} ({{ formatted_scalar_function_args_sql()}})
RETURNS {{ model.returns.data_type }}
{{ scalar_function_volatility_sql() }}
AS
{% endmacro %}

Expand All @@ -38,3 +39,18 @@
{{ model.compiled_code }}
$$ LANGUAGE SQL
{% endmacro %}

{% macro scalar_function_volatility_sql() %}
{{ return(adapter.dispatch('scalar_function_volatility_sql', 'dbt')()) }}
{% endmacro %}

{% macro default__scalar_function_volatility_sql() %}
{% if model.config.get('volatility') == 'deterministic' %}
IMMUTABLE
{% elif model.config.get('volatility') == 'stable' %}
STABLE
{% elif model.config.get('volatility') == 'non-deterministic' %}
VOLATILE
{% endif %}
{# If no volatility is set, don't add anything and let the data warehouse default it #}
{% endmacro %}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Under the Hood
body: Test that bigquery does nothing with volatility
time: 2025-10-29T16:56:03.92939-05:00
custom:
Author: QMalcolm
Issue: "1345"
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
{% macro bigquery__scalar_function_create_replace_signature_sql(target_relation) %}
CREATE OR REPLACE FUNCTION {{ target_relation.render() }} ({{ formatted_scalar_function_args_sql()}})
RETURNS {{ model.returns.data_type }}
{{ scalar_function_volatility_sql() }}
AS
{% endmacro %}

Expand All @@ -9,3 +10,10 @@
{{ model.compiled_code }}
)
{% endmacro %}

{% macro bigquery__scalar_function_volatility_sql() %}
{% set volatility = model.config.get('volatility') %}
{% if volatility != None %}
{% do exceptions.warn("`" ~ volatility ~ "` function volatility is not supported by BigQuery, and will be ignored") %}
{% endif %}
{% endmacro %}
147 changes: 145 additions & 2 deletions dbt-bigquery/tests/functional/functions/test_udfs.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,15 @@
import pytest
from dbt.tests.adapter.functions.test_udfs import UDFsBasic
from dbt.contracts.graph.nodes import FunctionNode
from dbt.contracts.results import RunStatus
from dbt.events.types import JinjaLogWarning
from dbt.tests.util import run_dbt
from dbt.tests.adapter.functions.test_udfs import (
UDFsBasic,
DeterministicUDF,
StableUDF,
NonDeterministicUDF,
)
from dbt_common.events.event_catcher import EventCatcher
from tests.functional.functions import files


Expand All @@ -12,4 +22,137 @@ def functions(self):
"price_for_xlarge.yml": files.MY_UDF_YML,
}

pass

class TestBigqueryDeterministicUDFs(DeterministicUDF):
@pytest.fixture(scope="class")
def functions(self):
return {
"price_for_xlarge.sql": files.MY_UDF_SQL,
"price_for_xlarge.yml": files.MY_UDF_YML,
}

def check_function_volatility(self, sql: str):
assert "VOLATILE" not in sql
assert "STABLE" not in sql
assert "IMMUTABLE" not in sql

def test_udfs(self, project, sql_event_catcher):
warning_event_catcher = EventCatcher(JinjaLogWarning)
result = run_dbt(
["build", "--debug"], callbacks=[sql_event_catcher.catch, warning_event_catcher.catch]
)

assert len(result.results) == 1
node_result = result.results[0]
assert node_result.status == RunStatus.Success
node = node_result.node
assert isinstance(node, FunctionNode)
assert node_result.node.name == "price_for_xlarge"

# Check volatility
assert len(sql_event_catcher.caught_events) == 1
self.check_function_volatility(sql_event_catcher.caught_events[0].data.sql)

# Check that the warning event was caught
assert len(warning_event_catcher.caught_events) == 1
assert (
"`deterministic` function volatility is not supported by BigQuery, and will be ignored"
in warning_event_catcher.caught_events[0].data.msg
)

# Check that the function can be executed
result = run_dbt(["show", "--inline", "SELECT {{ function('price_for_xlarge') }}(100)"])
assert len(result.results) == 1
# The result should have an agate table with one row and one column (and thus only one value, which is our inline selection)
select_value = int(result.results[0].agate_table.rows[0].values()[0])
assert select_value == 200 # the UDF should return 2x the input value (100 * 2 = 200)


class TestBigqueryStableUDFs(StableUDF):
@pytest.fixture(scope="class")
def functions(self):
return {
"price_for_xlarge.sql": files.MY_UDF_SQL,
"price_for_xlarge.yml": files.MY_UDF_YML,
}

def check_function_volatility(self, sql: str):
assert "VOLATILE" not in sql
assert "STABLE" not in sql
assert "IMMUTABLE" not in sql

def test_udfs(self, project, sql_event_catcher):
warning_event_catcher = EventCatcher(JinjaLogWarning)
result = run_dbt(
["build", "--debug"], callbacks=[sql_event_catcher.catch, warning_event_catcher.catch]
)

assert len(result.results) == 1
node_result = result.results[0]
assert node_result.status == RunStatus.Success
node = node_result.node
assert isinstance(node, FunctionNode)
assert node_result.node.name == "price_for_xlarge"

# Check volatility
assert len(sql_event_catcher.caught_events) == 1
self.check_function_volatility(sql_event_catcher.caught_events[0].data.sql)

# Check that the warning event was caught
assert len(warning_event_catcher.caught_events) == 1
assert (
"`stable` function volatility is not supported by BigQuery, and will be ignored"
in warning_event_catcher.caught_events[0].data.msg
)

# Check that the function can be executed
result = run_dbt(["show", "--inline", "SELECT {{ function('price_for_xlarge') }}(100)"])
assert len(result.results) == 1
# The result should have an agate table with one row and one column (and thus only one value, which is our inline selection)
select_value = int(result.results[0].agate_table.rows[0].values()[0])
assert select_value == 200 # the UDF should return 2x the input value (100 * 2 = 200)


class TestBigqueryNonDeterministicUDFs(NonDeterministicUDF):
@pytest.fixture(scope="class")
def functions(self):
return {
"price_for_xlarge.sql": files.MY_UDF_SQL,
"price_for_xlarge.yml": files.MY_UDF_YML,
}

def check_function_volatility(self, sql: str):
assert "VOLATILE" not in sql
assert "STABLE" not in sql
assert "IMMUTABLE" not in sql

def test_udfs(self, project, sql_event_catcher):
warning_event_catcher = EventCatcher(JinjaLogWarning)
result = run_dbt(
["build", "--debug"], callbacks=[sql_event_catcher.catch, warning_event_catcher.catch]
)

assert len(result.results) == 1
node_result = result.results[0]
assert node_result.status == RunStatus.Success
node = node_result.node
assert isinstance(node, FunctionNode)
assert node_result.node.name == "price_for_xlarge"

# Check volatility
assert len(sql_event_catcher.caught_events) == 1
self.check_function_volatility(sql_event_catcher.caught_events[0].data.sql)

# Check that the warning event was caught
assert len(warning_event_catcher.caught_events) == 1
assert (
"`non-deterministic` function volatility is not supported by BigQuery, and will be ignored"
in warning_event_catcher.caught_events[0].data.msg
)

# Check that the function can be executed
result = run_dbt(["show", "--inline", "SELECT {{ function('price_for_xlarge') }}(100)"])
assert len(result.results) == 1
# The result should have an agate table with one row and one column (and thus only one value, which is our inline selection)
select_value = int(result.results[0].agate_table.rows[0].values()[0])
assert select_value == 200 # the UDF should return 2x the input value (100 * 2 = 200)
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Support volatility setting for sql UDFs
time: 2025-10-29T16:41:09.298749-05:00
custom:
Author: QMalcolm
Issue: "1345"
19 changes: 18 additions & 1 deletion dbt-postgres/tests/functional/functions/test_udfs.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,22 @@
from dbt.tests.adapter.functions.test_udfs import UDFsBasic
from dbt.tests.adapter.functions.test_udfs import (
UDFsBasic,
DeterministicUDF,
StableUDF,
NonDeterministicUDF,
)


class TestPostgresUDFs(UDFsBasic):
pass


class TestPostgresDeterministicUDFs(DeterministicUDF):
pass


class TestPostgresStableUDFs(StableUDF):
pass


class TestPostgresNonDeterministicUDFs(NonDeterministicUDF):
pass
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Support volatility setting in sql UDFs
time: 2025-10-29T16:39:48.716884-05:00
custom:
Author: QMalcolm
Issue: "1345"
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
{% macro redshift__scalar_function_create_replace_signature_sql(target_relation) %}
CREATE OR REPLACE FUNCTION {{ target_relation.render() }} ({{ formatted_scalar_function_args_sql()}})
RETURNS {{ model.returns.data_type }}
{# TODO: Stop defaulting to VOLATILE once we have a way to set the volatility #}
{# We set a default here because redshift requires a volatility to be set #}
VOLATILE
{{ scalar_function_volatility_sql() }}
AS
{% endmacro %}

Expand All @@ -14,3 +12,14 @@
{%- endfor %}
{{ args | join(', ') }}
{% endmacro %}

{% macro redshift__scalar_function_volatility_sql() %}
{% if model.config.get('volatility') == 'deterministic' %}
IMMUTABLE
{% elif model.config.get('volatility') == 'stable' %}
STABLE
{% else %}
{# At this point, either they've set `non-deterministic` or they've set nothing. In either case, we default to VOLATILE #}
VOLATILE
{% endif %}
Copy link
Contributor

@colin-rogers-dbt colin-rogers-dbt Oct 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit:

Suggested change
{% if model.config.get('volatility') == 'deterministic' %}
IMMUTABLE
{% elif model.config.get('volatility') == 'stable' %}
STABLE
{% else %}
{# At this point, either they've set `non-deterministic` or they've set nothing. In either case, we default to VOLATILE #}
VOLATILE
{% endif %}
{% set volatility = model.config.get('volatility') %}
{% if volatility == 'deterministic' %}
IMMUTABLE
{% elif volatility == 'stable' %}
STABLE
{% else %}
{# At this point, either they've set `non-deterministic` or they've set nothing. In either case, we default to VOLATILE #}
VOLATILE
{% endif %}

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

also do we need to do any validation/warning if they have set something other than these options?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(also in the future if we add another option do we need to remember to come back and raise a warning?)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Core only accepts an Enum of possible volatility types deterministic, stable, and non-deterministic. So in order for a 4th option to come into existence, we'd first have to extend the enum in core. Additionally, have a hard time imagining any other volatility type being invented, but perhaps I'm being unimaginative 🤷🏻 Future proofing doesn't hurt anything.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay I think I've now added logic to sufficiently handle this 🙂

{% endmacro %}
41 changes: 37 additions & 4 deletions dbt-redshift/tests/functional/functions/test_udfs.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
from dbt.artifacts.schemas.results import RunStatus
from dbt.contracts.graph.nodes import FunctionNode
from dbt.tests.util import run_dbt
import pytest
from dbt.tests.adapter.functions.files import MY_UDF_YML
from dbt.tests.adapter.functions.test_udfs import UDFsBasic
from dbt.tests.adapter.functions.test_udfs import (
UDFsBasic,
DeterministicUDF,
StableUDF,
NonDeterministicUDF,
)
from tests.functional.functions.files import MY_UDF_SQL


Expand All @@ -15,3 +17,34 @@ def functions(self):
"price_for_xlarge.sql": MY_UDF_SQL,
"price_for_xlarge.yml": MY_UDF_YML,
}

def check_function_volatility(self, sql: str):
# in redshift, if no volatility is set, we template in VOLATILE
assert "VOLATILE" in sql


class TestRedshiftDeterministicUDFs(DeterministicUDF):
@pytest.fixture(scope="class")
def functions(self):
return {
"price_for_xlarge.sql": MY_UDF_SQL,
"price_for_xlarge.yml": MY_UDF_YML,
}


class TestRedshiftStableUDFs(StableUDF):
@pytest.fixture(scope="class")
def functions(self):
return {
"price_for_xlarge.sql": MY_UDF_SQL,
"price_for_xlarge.yml": MY_UDF_YML,
}


class TestRedshiftNonDeterministicUDFs(NonDeterministicUDF):
@pytest.fixture(scope="class")
def functions(self):
return {
"price_for_xlarge.sql": MY_UDF_SQL,
"price_for_xlarge.yml": MY_UDF_YML,
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Support volatility setting in sql UDFs
time: 2025-10-29T16:40:24.708639-05:00
custom:
Author: QMalcolm
Issue: "1345"
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
CREATE OR REPLACE FUNCTION {{ target_relation.render() }} ({{ formatted_scalar_function_args_sql()}})
RETURNS {{ model.returns.data_type }}
LANGUAGE SQL
{{ scalar_function_volatility_sql() }}
AS
{% endmacro %}

Expand All @@ -10,3 +11,14 @@
{{ model.compiled_code }}
$$
{% endmacro %}

{% macro snowflake__scalar_function_volatility_sql() %}
{% if model.config.get('volatility') == 'deterministic' %}
IMMUTABLE
{% elif model.config.get('volatility') == 'stable' %}
{% do exceptions.warn("`Stable` function volatility is not supported by Snowflake, and will be ignored") %}
{% elif model.config.get('volatility') == 'non-deterministic' %}
VOLATILE
{% endif %}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this more future proof?

Suggested change
{% if model.config.get('volatility') == 'deterministic' %}
IMMUTABLE
{% elif model.config.get('volatility') == 'stable' %}
{% do exceptions.warn("`Stable` function volatility is not supported by Snowflake, and will be ignored") %}
{% elif model.config.get('volatility') == 'non-deterministic' %}
VOLATILE
{% endif %}
{% set volatility = model.config.get('volatility') %}
{% if volatility == 'deterministic' %}
IMMUTABLE
{% elif model.config.get('volatility') == 'non-deterministic' %}
VOLATILE
{% elif volatility is not none %}
{% do exceptions.warn("user passed unsupported volatility config value: " ~ volatility ~"; it will be ignored") %}
{% endif %}

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay I think I've now added logic to sufficiently handle this 🙂

{# If no volatility is set, don't add anything and let the data warehouse default it #}
{% endmacro %}
Loading
Loading