diff --git a/.changes/unreleased/Features-20250627-094000.yaml b/.changes/unreleased/Features-20250627-094000.yaml new file mode 100644 index 00000000000..f7160c1457c --- /dev/null +++ b/.changes/unreleased/Features-20250627-094000.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Let ConfigSelectorMethod search through all nodes to return matching selections +time: 2025-06-27T09:40:00.485654-05:00 +custom: + Author: trouze + Issue: "11607" diff --git a/core/dbt/graph/selector_methods.py b/core/dbt/graph/selector_methods.py index 41a9cc0d51c..966daf9bdd6 100644 --- a/core/dbt/graph/selector_methods.py +++ b/core/dbt/graph/selector_methods.py @@ -22,7 +22,6 @@ ManifestNode, Metric, ModelNode, - ResultNode, SavedQuery, SemanticModel, SingularTestNode, @@ -205,11 +204,6 @@ def all_nodes( self.saved_query_nodes(included_nodes), ) - def configurable_nodes( - self, included_nodes: Set[UniqueId] - ) -> Iterator[Tuple[UniqueId, ResultNode]]: - yield from chain(self.parsed_nodes(included_nodes), self.source_nodes(included_nodes)) - def non_source_nodes( self, included_nodes: Set[UniqueId], @@ -536,7 +530,7 @@ def search( # search sources is kind of useless now source configs only have # 'enabled', which you can't really filter on anyway, but maybe we'll # add more someday, so search them anyway. - for unique_id, node in self.configurable_nodes(included_nodes): + for unique_id, node in self.all_nodes(included_nodes): try: value = _getattr_descend(node.config, parts) except AttributeError: diff --git a/tests/functional/graph_selection/fixtures.py b/tests/functional/graph_selection/fixtures.py index b4d4a677d25..7c58039e1c0 100644 --- a/tests/functional/graph_selection/fixtures.py +++ b/tests/functional/graph_selection/fixtures.py @@ -109,7 +109,8 @@ {{ config( materialized = 'table', - tags=['bi', 'users'] + tags=['bi', 'users'], + meta={'contains_pii':true} ) }} diff --git a/tests/functional/graph_selection/test_config_selection.py b/tests/functional/graph_selection/test_config_selection.py new file mode 100644 index 00000000000..efd7829aa3c --- /dev/null +++ b/tests/functional/graph_selection/test_config_selection.py @@ -0,0 +1,140 @@ +""" +Functional tests for config selector functionality. + +These tests validate that config selectors work correctly after the change from +configurable_nodes() to all_nodes() in ConfigSelectorMethod. +""" + +import pytest + +from dbt.tests.util import run_dbt +from tests.functional.graph_selection.fixtures import ( + alternative_users_sql, + base_users_sql, + emails_alt_sql, + emails_sql, + nested_users_sql, + never_selected_sql, + schema_yml, + subdir_sql, + users_rollup_dependency_sql, + users_rollup_sql, + users_sql, +) + + +class TestConfigSelection: + """Test config selectors work on all node types including newly supported ones.""" + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": schema_yml, + "base_users.sql": base_users_sql, + "users.sql": users_sql, + "users_rollup.sql": users_rollup_sql, + "versioned_v3.sql": base_users_sql, + "users_rollup_dependency.sql": users_rollup_dependency_sql, + "emails.sql": emails_sql, + "emails_alt.sql": emails_alt_sql, + "alternative.users.sql": alternative_users_sql, + "never_selected.sql": never_selected_sql, + "test": { + "subdir.sql": subdir_sql, + "subdir": {"nested_users.sql": nested_users_sql}, + }, + "metricflow_time_spine.sql": "SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day", + "semantic_models.yml": """ +version: 2 + +semantic_models: + - name: test_semantic_model + label: "Test Semantic Model" + model: ref('users') + dimensions: + - name: created_at + type: time + type_params: + time_granularity: day + measures: + - name: total_count + agg: count + expr: 1 + entities: + - name: id + type: primary + defaults: + agg_time_dimension: created_at + +metrics: + - name: test_metric + label: "Test Metric" + type: simple + config: + enabled: true + meta: + metric_type: simple + type_params: + measure: total_count + +saved_queries: + - name: test_saved_query + label: "Test Saved Query" + config: + enabled: true + meta: + query_type: basic + contains_pii: true + query_params: + metrics: + - test_metric + group_by: + - "Dimension('test_semantic_model__created_at')" + exports: + - name: test_export + config: + alias: test_export_alias + export_as: table +""", + } + + def test_config_selector_with_resource_type_filter(self, project): + """Test config selectors with resource type filters.""" + + results = run_dbt(["list", "--resource-type", "model", "--select", "config.enabled:true"]) + selected_nodes = set(results) + + assert "saved_query:test.test_saved_query" not in selected_nodes + assert "metric:test.test_metric" not in selected_nodes + assert "semantic_model:test.test_semantic_model" not in selected_nodes + + def test_config_enabled_true_selects_extended_nodes(self, project): + """Test that dbt ls -s config.enabled:true returns the test_saved_query. + + This specific test validates that the saved query (which has config.enabled:true) + is properly selected by the config selector. This demonstrates that the change from + configurable_nodes() to all_nodes() allows config selectors to work on saved queries. + """ + + results = run_dbt(["list", "--select", "config.enabled:true"]) + selected_nodes = set(results) + + assert "saved_query:test.test_saved_query" in selected_nodes + assert "metric:test.test_metric" in selected_nodes + assert "semantic_model:test.test_semantic_model" in selected_nodes + + def test_config_meta_selection(self, project): + """ """ + + results = run_dbt(["list", "--select", "config.meta.contains_pii:true"]) + selected_nodes = set(results) + + assert "test.users" in selected_nodes + assert "saved_query:test.test_saved_query" in selected_nodes + assert "test.unique_users_id" in selected_nodes diff --git a/tests/unit/graph/test_selector_methods.py b/tests/unit/graph/test_selector_methods.py index d500c631a1b..e833048bd1c 100644 --- a/tests/unit/graph/test_selector_methods.py +++ b/tests/unit/graph/test_selector_methods.py @@ -427,6 +427,128 @@ def test_select_config_meta(manifest): assert not search_manifest_using_method(manifest, list_method, "other") == {"table_model"} +def test_select_config_enabled_on_all_nodes(manifest): + """Test that ConfigSelectorMethod can find enabled property on all node types. + + This test verifies that the change from configurable_nodes() to all_nodes() + allows the ConfigSelectorMethod to search through all node types, not just + the previous subset of parsed_nodes and source_nodes. + """ + methods = MethodManager(manifest, None) + method = methods.get_method("config", ["enabled"]) + assert isinstance(method, ConfigSelectorMethod) + assert method.arguments == ["enabled"] + + # All nodes should have enabled: true by default + enabled_nodes = search_manifest_using_method(manifest, method, "true") + + # Should include traditional configurable nodes (models, sources) + assert "table_model" in enabled_nodes + assert "view_model" in enabled_nodes + assert "ext_raw.ext_source" in enabled_nodes + + # Should now also include the new node types that were added via all_nodes() + # Metrics, semantic models, saved queries should all have enabled: true by default + assert "my_metric" in enabled_nodes + assert "test_semantic_model" in enabled_nodes + assert "test_saved_query" in enabled_nodes + + +def test_select_config_searches_more_nodes_than_before(manifest): + """Test that ConfigSelectorMethod now searches more node types than before. + + This test verifies that the change from configurable_nodes() to all_nodes() + actually expands the set of searchable nodes beyond just models and sources. + """ + methods = MethodManager(manifest, None) + + # Test that we can search for config properties that exist on various node types + enabled_method = methods.get_method("config", ["enabled"]) + all_enabled_nodes = search_manifest_using_method(manifest, enabled_method, "true") + + # Verify specific new node types are found (these were not in configurable_nodes()) + metrics = [node for node in all_enabled_nodes if "my_metric" in node] + semantic_models = [node for node in all_enabled_nodes if "test_semantic_model" in node] + saved_queries = [node for node in all_enabled_nodes if "test_saved_query" in node] + + # With all_nodes(), we should find these new node types + assert len(metrics) > 0, "Should find metric nodes (new with all_nodes)" + assert len(semantic_models) > 0, "Should find semantic model nodes (new with all_nodes)" + assert len(saved_queries) > 0, "Should find saved query nodes (new with all_nodes)" + + # Specifically check that these new node types are found + assert "my_metric" in all_enabled_nodes, "Metric should be searchable with config selector" + assert ( + "test_semantic_model" in all_enabled_nodes + ), "Semantic model should be searchable with config selector" + assert ( + "test_saved_query" in all_enabled_nodes + ), "Saved query should be searchable with config selector" + + # The total should include many types of nodes due to all_nodes() expansion + total_nodes = len(all_enabled_nodes) + assert ( + total_nodes > 20 + ), f"Should find many nodes with all_nodes() expansion, found {total_nodes}" + + +def test_config_selector_finds_nodes_not_in_old_configurable_nodes(manifest): + """Test that ConfigSelectorMethod now finds nodes that were excluded by configurable_nodes(). + + Before the change, ConfigSelectorMethod.search() used configurable_nodes() which only + included parsed_nodes() and source_nodes(). Now it uses all_nodes() which includes + exposure_nodes(), metric_nodes(), unit_tests(), semantic_model_nodes(), and saved_query_nodes(). + + This test verifies that the new node types are now searchable. + """ + methods = MethodManager(manifest, None) + method = methods.get_method("config", ["enabled"]) + + # Search for enabled nodes + enabled_nodes = search_manifest_using_method(manifest, method, "true") + + # These node types were NOT included in the old configurable_nodes() method + # but should now be found via all_nodes() + + # Metrics should be found (they have MetricConfig with enabled property) + # search_name for metrics is just the metric name, not the full unique_id + assert ( + "my_metric" in enabled_nodes + ), "Specific metric should be found by config selector (new functionality)" + + # Semantic models should be found (they have SemanticModelConfig with enabled property) + assert ( + "test_semantic_model" in enabled_nodes + ), "Specific semantic model should be found by config selector (new functionality)" + + # Saved queries should be found (they have SavedQueryConfig with enabled property) + assert ( + "test_saved_query" in enabled_nodes + ), "Specific saved query should be found by config selector (new functionality)" + + # Unit tests should also be found (they have config.enabled) + unit_tests_found = [node for node in enabled_nodes if "unit_test" in node] + assert ( + len(unit_tests_found) > 0 + ), "Unit tests should be found by config selector (new functionality)" + + # Count the new node types we can find + new_node_types = 0 + if "my_metric" in enabled_nodes: + new_node_types += 1 + if "test_semantic_model" in enabled_nodes: + new_node_types += 1 + if "test_saved_query" in enabled_nodes: + new_node_types += 1 + new_node_types += len(unit_tests_found) + + # This demonstrates the key change: ConfigSelectorMethod can now search through + # many more node types than just the traditional "configurable" nodes + assert ( + new_node_types >= 4 + ), f"Should find at least 4 nodes from newly included types, found {new_node_types}" + + def test_select_test_name(manifest): methods = MethodManager(manifest, None) method = methods.get_method("test_name", [])