Skip to content

Commit 23ee61c

Browse files
authored
fix(oso_dagster): hotfix polars io manager with bq (#4672)
1 parent 7664134 commit 23ee61c

File tree

4 files changed

+47
-2
lines changed

4 files changed

+47
-2
lines changed

pyproject.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ dependencies = [
6767
"gcloud-aio-storage<10.0.0,>=9.3.0",
6868
"kr8s==0.20.9",
6969
"structlog>=25.4.0",
70+
"pandas-gbq>=0.29.2",
7071
]
7172
name = "oso"
7273
version = "1.0.0"

uv.lock

Lines changed: 38 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

warehouse/oso_dagster/definitions/resources.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
import logging
21
import typing as t
32

3+
import structlog
44
from dagster import ConfigurableIOManagerFactory
55
from dagster_dlt import DagsterDltResource
66
from dagster_gcp import BigQueryResource, GCSResource
@@ -45,7 +45,7 @@
4545
from ..config import DagsterConfig
4646
from ..utils import GCPSecretResolver, LocalSecretResolver
4747

48-
logger = logging.getLogger(__name__)
48+
logger = structlog.get_logger(__name__)
4949

5050

5151
@resource_factory("global_config")
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,15 @@
1+
import structlog
12
from dagster_duckdb_polars import DuckDBPolarsIOManager
23
from dagster_polars import PolarsBigQueryIOManager
34
from oso_dagster.config import DagsterConfig
45

6+
logger = structlog.get_logger(__name__)
7+
58

69
def load_io_manager(global_config: DagsterConfig):
710
if global_config.enable_bigquery:
11+
logger.info("Using PolarsBigQueryIOManager")
812
return PolarsBigQueryIOManager(project=global_config.project_id)
13+
14+
logger.info("Using DuckDBPolarsIOManager")
915
return DuckDBPolarsIOManager(database=global_config.local_duckdb_path)

0 commit comments

Comments
 (0)