Skip to content

Commit b187b6d

Browse files
committed
fix: Infer entity value types from dbt column types
When creating entities from dbt models with integer columns, entities were being created with default ValueType.STRING, causing validation errors: "Entity X has type ValueType.STRING, which does not match the inferred type Int64" Solution: Added mapping dict and helper function for clean type conversion: - FEAST_TYPE_TO_VALUE_TYPE: Maps FeastType to ValueType - feast_type_to_value_type(): Helper function for conversion - _infer_entity_value_type(): Method in DbtToFeastMapper class This replaces verbose if-else chains with a clean dictionary lookup pattern. Also corrects schema generation to include entity columns, as FeatureView.__init__ expects to extract entity columns from the schema itself. Signed-off-by: yassinnouh21 <[email protected]>
1 parent ad1848a commit b187b6d

19 files changed

+1571
-56
lines changed

sdk/python/feast/cli/dbt_import.py

Lines changed: 2 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -225,27 +225,8 @@ def import_command(
225225
model_entities: List[Any] = []
226226
for entity_col in entity_cols:
227227
if entity_col not in entities_created:
228-
# Infer entity value type from model column
229-
from feast.types import String, Int32, Int64, Float32, Float64, Bool
230-
from feast.value_type import ValueType
231-
from feast.dbt.mapper import map_dbt_type_to_feast_type
232-
233-
entity_value_type = ValueType.UNKNOWN
234-
for column in model.columns:
235-
if column.name == entity_col:
236-
feast_type = map_dbt_type_to_feast_type(column.data_type)
237-
if feast_type == String:
238-
entity_value_type = ValueType.STRING
239-
elif feast_type in [Int32, Int64]:
240-
entity_value_type = ValueType.INT64
241-
elif feast_type in [Float32, Float64]:
242-
entity_value_type = ValueType.DOUBLE
243-
elif feast_type == Bool:
244-
entity_value_type = ValueType.BOOL
245-
else:
246-
entity_value_type = ValueType.STRING
247-
break
248-
228+
# Use mapper's internal method for value type inference
229+
entity_value_type = mapper._infer_entity_value_type(model, entity_col)
249230
entity = mapper.create_entity(
250231
name=entity_col,
251232
description="Entity key for dbt models",

sdk/python/feast/dbt/mapper.py

Lines changed: 28 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,24 @@
2626
)
2727
from feast.value_type import ValueType
2828

29+
# Mapping from FeastType to ValueType for entity value inference
30+
FEAST_TYPE_TO_VALUE_TYPE: Dict[FeastType, ValueType] = {
31+
String: ValueType.STRING,
32+
Int32: ValueType.INT64,
33+
Int64: ValueType.INT64,
34+
Float32: ValueType.DOUBLE,
35+
Float64: ValueType.DOUBLE,
36+
Bool: ValueType.BOOL,
37+
Bytes: ValueType.BYTES,
38+
UnixTimestamp: ValueType.UNIX_TIMESTAMP,
39+
}
40+
41+
42+
def feast_type_to_value_type(feast_type: FeastType) -> ValueType:
43+
"""Convert a FeastType to its corresponding ValueType for entities."""
44+
return FEAST_TYPE_TO_VALUE_TYPE.get(feast_type, ValueType.STRING)
45+
46+
2947
# Comprehensive mapping from dbt/warehouse types to Feast types
3048
# Covers BigQuery, Snowflake, Redshift, PostgreSQL, and common SQL types
3149
DBT_TO_FEAST_TYPE_MAP: Dict[str, FeastType] = {
@@ -180,6 +198,14 @@ def __init__(
180198
self.timestamp_field = timestamp_field
181199
self.ttl_days = ttl_days
182200

201+
def _infer_entity_value_type(self, model: DbtModel, entity_col: str) -> ValueType:
202+
"""Infer entity ValueType from dbt model column type."""
203+
for column in model.columns:
204+
if column.name == entity_col:
205+
feast_type = map_dbt_type_to_feast_type(column.data_type)
206+
return feast_type_to_value_type(feast_type)
207+
return ValueType.UNKNOWN
208+
183209
def create_data_source(
184210
self,
185211
model: DbtModel,
@@ -357,23 +383,7 @@ def create_feature_view(
357383
entity_objs = []
358384
for entity_col in entity_cols:
359385
# Infer entity value type from model column
360-
entity_value_type = ValueType.UNKNOWN
361-
for column in model.columns:
362-
if column.name == entity_col:
363-
feast_type = map_dbt_type_to_feast_type(column.data_type)
364-
# Convert Feast type to ValueType
365-
if feast_type == String:
366-
entity_value_type = ValueType.STRING
367-
elif feast_type in [Int32, Int64]:
368-
entity_value_type = ValueType.INT64
369-
elif feast_type in [Float32, Float64]:
370-
entity_value_type = ValueType.DOUBLE
371-
elif feast_type == Bool:
372-
entity_value_type = ValueType.BOOL
373-
else:
374-
entity_value_type = ValueType.STRING
375-
break
376-
386+
entity_value_type = self._infer_entity_value_type(model, entity_col)
377387
ent = self.create_entity(
378388
name=entity_col,
379389
description=f"Entity for {model.name}",
@@ -436,24 +446,7 @@ def create_all_from_model(
436446
# Create entities (plural)
437447
entities_list = []
438448
for entity_col in entity_cols:
439-
# Infer entity value type from model column
440-
entity_value_type = ValueType.UNKNOWN
441-
for column in model.columns:
442-
if column.name == entity_col:
443-
feast_type = map_dbt_type_to_feast_type(column.data_type)
444-
# Convert Feast type to ValueType
445-
if feast_type == String:
446-
entity_value_type = ValueType.STRING
447-
elif feast_type in [Int32, Int64]:
448-
entity_value_type = ValueType.INT64
449-
elif feast_type in [Float32, Float64]:
450-
entity_value_type = ValueType.DOUBLE
451-
elif feast_type == Bool:
452-
entity_value_type = ValueType.BOOL
453-
else:
454-
entity_value_type = ValueType.STRING
455-
break
456-
449+
entity_value_type = self._infer_entity_value_type(model, entity_col)
457450
entity = self.create_entity(
458451
name=entity_col,
459452
description=f"Entity for {model.name}",
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"""dbt integration tests."""
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
"""
2+
Conftest for dbt integration tests.
3+
4+
This is a standalone conftest that doesn't depend on the main Feast test infrastructure.
5+
"""
6+
7+
from pathlib import Path
8+
9+
import pytest
10+
11+
# This conftest is minimal and doesn't import the main feast conftest
12+
# to avoid complex dependency chains for dbt-specific tests
13+
14+
# Path to the test dbt project manifest
15+
TEST_DBT_PROJECT_DIR = Path(__file__).parent / "test_dbt_project"
16+
TEST_MANIFEST_PATH = TEST_DBT_PROJECT_DIR / "target" / "manifest.json"
17+
18+
19+
def pytest_collection_modifyitems(config, items): # noqa: ARG001
20+
"""
21+
Skip dbt integration tests if manifest.json doesn't exist.
22+
23+
These tests require running 'dbt build' first to generate the manifest.
24+
The dbt-integration-test workflow handles this, but regular unit test
25+
runs don't, so we skip them to avoid failures.
26+
"""
27+
if not TEST_MANIFEST_PATH.exists():
28+
skip_marker = pytest.mark.skip(
29+
reason="dbt manifest.json not found - run 'dbt build' first or use dbt-integration-test workflow"
30+
)
31+
for item in items:
32+
item.add_marker(skip_marker)
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
[pytest]
2+
# Prevent loading parent conftest.py files which may import heavy dependencies
3+
# like ray that are not needed for dbt integration tests
4+
norecursedirs = ..
5+
asyncio_mode = auto
6+
7+
# Test markers
8+
markers =
9+
dbt: dbt integration tests

0 commit comments

Comments
 (0)