Skip to content

Commit 9e84e1c

Browse files
devin-ai-integration[bot]aaronsteers
devin-ai-integration[bot]
and
aaronsteers
committed
fix: update entrypoint to make --config optional for discovery
Co-Authored-By: Aaron <AJ> Steers <[email protected]>
1 parent 7490ad1 commit 9e84e1c

File tree

3 files changed

+131
-28
lines changed

3 files changed

+131
-28
lines changed

‎airbyte_cdk/entrypoint.py‎

Lines changed: 35 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ def parse_args(args: List[str]) -> argparse.Namespace:
9393
)
9494
required_discover_parser = discover_parser.add_argument_group("required named arguments")
9595
required_discover_parser.add_argument(
96-
"--config", type=str, required=True, help="path to the json configuration file"
96+
"--config", type=str, required=False, help="path to the json configuration file"
9797
)
9898

9999
# read
@@ -147,33 +147,44 @@ def run(self, parsed_args: argparse.Namespace) -> Iterable[str]:
147147
]
148148
yield self.airbyte_message_to_string(message)
149149
else:
150-
raw_config = self.source.read_config(parsed_args.config)
151-
config = self.source.configure(raw_config, temp_dir)
152-
153-
yield from [
154-
self.airbyte_message_to_string(queued_message)
155-
for queued_message in self._emit_queued_messages(self.source)
156-
]
157-
if cmd == "check":
158-
yield from map(
159-
AirbyteEntrypoint.airbyte_message_to_string,
160-
self.check(source_spec, config),
161-
)
162-
elif cmd == "discover":
163-
yield from map(
164-
AirbyteEntrypoint.airbyte_message_to_string,
165-
self.discover(source_spec, config),
166-
)
167-
elif cmd == "read":
168-
config_catalog = self.source.read_catalog(parsed_args.catalog)
169-
state = self.source.read_state(parsed_args.state)
170-
150+
if cmd == "discover" and not parsed_args.config and not self.source.check_config_against_spec:
151+
empty_config = {}
152+
yield from [
153+
self.airbyte_message_to_string(queued_message)
154+
for queued_message in self._emit_queued_messages(self.source)
155+
]
171156
yield from map(
172157
AirbyteEntrypoint.airbyte_message_to_string,
173-
self.read(source_spec, config, config_catalog, state),
158+
self.discover(source_spec, empty_config),
174159
)
175160
else:
176-
raise Exception("Unexpected command " + cmd)
161+
raw_config = self.source.read_config(parsed_args.config)
162+
config = self.source.configure(raw_config, temp_dir)
163+
164+
yield from [
165+
self.airbyte_message_to_string(queued_message)
166+
for queued_message in self._emit_queued_messages(self.source)
167+
]
168+
if cmd == "check":
169+
yield from map(
170+
AirbyteEntrypoint.airbyte_message_to_string,
171+
self.check(source_spec, config),
172+
)
173+
elif cmd == "discover":
174+
yield from map(
175+
AirbyteEntrypoint.airbyte_message_to_string,
176+
self.discover(source_spec, config),
177+
)
178+
elif cmd == "read":
179+
config_catalog = self.source.read_catalog(parsed_args.catalog)
180+
state = self.source.read_state(parsed_args.state)
181+
182+
yield from map(
183+
AirbyteEntrypoint.airbyte_message_to_string,
184+
self.read(source_spec, config, config_catalog, state),
185+
)
186+
else:
187+
raise Exception("Unexpected command " + cmd)
177188
finally:
178189
yield from [
179190
self.airbyte_message_to_string(queued_message)

‎unit_tests/sources/declarative/test_manifest_declarative_source_dynamic_schema.py‎

Lines changed: 95 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
#
22
#
33

4-
from unittest.mock import MagicMock
4+
from unittest.mock import MagicMock, patch
55

66
import pytest
7-
7+
from airbyte_cdk.models import AirbyteCatalog
88
from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
9+
from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit
910

1011

1112
def test_check_config_against_spec_with_dynamic_schema_loader():
@@ -63,4 +64,96 @@ def test_check_config_against_spec_without_dynamic_schema_loader():
6364

6465
source = ManifestDeclarativeSource(source_config=source_config)
6566

67+
68+
@patch("airbyte_cdk.sources.declarative.manifest_declarative_source.ManifestDeclarativeSource.streams")
69+
def test_discover_with_dynamic_schema_loader_no_config(mock_streams):
70+
"""Test that discovery works without config when DynamicSchemaLoader is used."""
71+
mock_stream = MagicMock()
72+
mock_stream.name = "test_dynamic_stream"
73+
74+
mock_airbyte_stream = MagicMock()
75+
type(mock_airbyte_stream).name = "test_dynamic_stream"
76+
mock_stream.as_airbyte_stream.return_value = mock_airbyte_stream
77+
78+
mock_streams.return_value = [mock_stream]
79+
80+
source_config = {
81+
"type": "DeclarativeSource",
82+
"check": {"type": "CheckStream"},
83+
"streams": [
84+
{
85+
"name": "test_dynamic_stream",
86+
"schema_loader": {
87+
"type": "DynamicSchemaLoader",
88+
"retriever": {
89+
"type": "SimpleRetriever",
90+
"requester": {"url_base": "https://example.com", "http_method": "GET"},
91+
"record_selector": {"extractor": {"field_path": []}},
92+
},
93+
"schema_type_identifier": {
94+
"key_pointer": ["name"],
95+
},
96+
},
97+
"retriever": {
98+
"type": "SimpleRetriever",
99+
"requester": {"url_base": "https://example.com", "http_method": "GET"},
100+
"record_selector": {"extractor": {"field_path": []}},
101+
},
102+
}
103+
],
104+
"version": "0.1.0",
105+
}
106+
107+
source = ManifestDeclarativeSource(source_config=source_config)
108+
109+
assert source.check_config_against_spec is False
110+
111+
logger = MagicMock()
112+
catalog = source.discover(logger, {})
113+
114+
assert isinstance(catalog, AirbyteCatalog)
115+
assert len(catalog.streams) == 1
116+
assert catalog.streams[0].name == "test_dynamic_stream"
117+
118+
119+
@patch("airbyte_cdk.sources.declarative.manifest_declarative_source.ManifestDeclarativeSource.streams")
120+
def test_discover_without_dynamic_schema_loader_no_config(mock_streams):
121+
"""Test that discovery validates config when DynamicSchemaLoader is not used."""
122+
mock_stream = MagicMock()
123+
mock_stream.name = "test_static_stream"
124+
125+
mock_airbyte_stream = MagicMock()
126+
type(mock_airbyte_stream).name = "test_static_stream"
127+
mock_stream.as_airbyte_stream.return_value = mock_airbyte_stream
128+
129+
mock_streams.return_value = [mock_stream]
130+
131+
source_config = {
132+
"type": "DeclarativeSource",
133+
"check": {"type": "CheckStream"},
134+
"streams": [
135+
{
136+
"name": "test_static_stream",
137+
"schema_loader": {"type": "InlineSchemaLoader", "schema": {}},
138+
"retriever": {
139+
"type": "SimpleRetriever",
140+
"requester": {"url_base": "https://example.com", "http_method": "GET"},
141+
"record_selector": {"extractor": {"field_path": []}},
142+
},
143+
}
144+
],
145+
"version": "0.1.0",
146+
}
147+
148+
source = ManifestDeclarativeSource(source_config=source_config)
149+
150+
assert source.check_config_against_spec is True
151+
152+
logger = MagicMock()
153+
catalog = source.discover(logger, {})
154+
155+
assert isinstance(catalog, AirbyteCatalog)
156+
assert len(catalog.streams) == 1
157+
assert catalog.streams[0].name == "test_static_stream"
158+
66159
assert source.check_config_against_spec is True

‎unit_tests/test_entrypoint.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,14 +172,13 @@ def test_parse_valid_args(
172172
["cmd", "args"],
173173
[
174174
("check", {"config": "config_path"}),
175-
("discover", {"config": "config_path"}),
176175
("read", {"config": "config_path", "catalog": "catalog_path"}),
177176
],
178177
)
179178
def test_parse_missing_required_args(
180179
cmd: str, args: MutableMapping[str, Any], entrypoint: AirbyteEntrypoint
181180
):
182-
required_args = {"check": ["config"], "discover": ["config"], "read": ["config", "catalog"]}
181+
required_args = {"check": ["config"], "read": ["config", "catalog"]}
183182
for required_arg in required_args[cmd]:
184183
argcopy = deepcopy(args)
185184
del argcopy[required_arg]

0 commit comments

Comments
 (0)