Skip to content

Commit f5f7b03

Browse files
Fix: Instead of failing, raise a warning if stream selection is performed before config is present (#285)
1 parent 2682ed2 commit f5f7b03

File tree

1 file changed

+27
-0
lines changed

1 file changed

+27
-0
lines changed

airbyte/sources/base.py

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ def __init__(
8181
self._discovered_catalog: AirbyteCatalog | None = None
8282
self._spec: ConnectorSpecification | None = None
8383
self._selected_stream_names: list[str] = []
84+
self._to_be_selected_streams: list[str] | str = []
8485
if config is not None:
8586
self.set_config(config, validate=validate)
8687
if streams is not None:
@@ -100,12 +101,29 @@ def set_streams(self, streams: list[str]) -> None:
100101
)
101102
self.select_streams(streams)
102103

104+
def _log_warning_preselected_stream(self, streams: str | list[str]) -> None:
105+
"""Logs a warning message indicating stream selection which are not selected yet"""
106+
if streams == "*":
107+
print(
108+
"Warning: Config is not set yet. All streams will be selected after config is set."
109+
)
110+
else:
111+
print(
112+
"Warning: Config is not set yet. "
113+
f"Streams to be selected after config is set: {streams}"
114+
)
115+
103116
def select_all_streams(self) -> None:
104117
"""Select all streams.
105118
106119
This is a more streamlined equivalent to:
107120
> source.select_streams(source.get_available_streams()).
108121
"""
122+
if self._config_dict is None:
123+
self._to_be_selected_streams = "*"
124+
self._log_warning_preselected_stream(self._to_be_selected_streams)
125+
return
126+
109127
self._selected_stream_names = self.get_available_streams()
110128

111129
def select_streams(self, streams: str | list[str]) -> None:
@@ -116,6 +134,11 @@ def select_streams(self, streams: str | list[str]) -> None:
116134
117135
Currently, if this is not set, all streams will be read.
118136
"""
137+
if self._config_dict is None:
138+
self._to_be_selected_streams = streams
139+
self._log_warning_preselected_stream(streams)
140+
return
141+
119142
if streams == "*":
120143
self.select_all_streams()
121144
return
@@ -159,6 +182,10 @@ def set_config(
159182

160183
self._config_dict = config
161184

185+
if self._to_be_selected_streams:
186+
self.select_streams(self._to_be_selected_streams)
187+
self._to_be_selected_streams = []
188+
162189
def get_config(self) -> dict[str, Any]:
163190
"""Get the config for the connector."""
164191
return self._config

0 commit comments

Comments
 (0)