Skip to content

Commit 9472e09

Browse files
committed
Add unit test
1 parent 679b3ca commit 9472e09

File tree

1 file changed

+90
-0
lines changed

1 file changed

+90
-0
lines changed

unit_tests/sources/declarative/resolvers/test_config_components_resolver.py

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
import json
66
from copy import deepcopy
7+
from datetime import date as dt_date
78
from unittest.mock import MagicMock
89

910
import pytest
@@ -386,3 +387,92 @@ def test_component_mapping_conditions(manifest, config, expected_conditional_par
386387
stream._stream_partition_generator._partition_factory._retriever.requester._parameters
387388
== expected_conditional_params[stream.name]
388389
)
390+
391+
392+
_MANIFEST_WITH_VALUE_TYPE_STR = deepcopy(_MANIFEST)
393+
_MANIFEST_WITH_VALUE_TYPE_STR["dynamic_streams"][0]["components_resolver"]["components_mapping"].extend(
394+
[
395+
{
396+
"type": "ComponentMappingDefinition",
397+
"field_path": ["retriever", "requester", "$parameters", "as_string"],
398+
"value": "true", # valid YAML, but we want to keep it as *string*
399+
"value_type": "string",
400+
"create_or_update": True,
401+
},
402+
{
403+
"type": "ComponentMappingDefinition",
404+
"field_path": ["retriever", "requester", "$parameters", "as_yaml"],
405+
"value": "true", # no value_type -> should be parsed to boolean True
406+
"create_or_update": True,
407+
},
408+
{
409+
"type": "ComponentMappingDefinition",
410+
"field_path": ["retriever", "requester", "$parameters", "json_string"],
411+
"value": "[1, 2]", # valid YAML/JSON-looking text; keep as string
412+
"value_type": "string",
413+
"create_or_update": True,
414+
},
415+
{
416+
"type": "ComponentMappingDefinition",
417+
"field_path": ["retriever", "requester", "$parameters", "json_parsed"],
418+
"value": "[1, 2]", # no value_type -> should parse to a list
419+
"create_or_update": True,
420+
},
421+
{
422+
"type": "ComponentMappingDefinition",
423+
"field_path": ["retriever", "requester", "$parameters", "date_as_string"],
424+
"value": "2024-07-10", # date-like text that YAML would parse; force keep as string
425+
"value_type": "string",
426+
"create_or_update": True,
427+
},
428+
{
429+
"type": "ComponentMappingDefinition",
430+
"field_path": ["retriever", "requester", "$parameters", "date_yaml_parsed"],
431+
"value": "2024-07-10", # no value_type -> YAML should parse to datetime.date
432+
"create_or_update": True,
433+
}
434+
]
435+
)
436+
437+
438+
def test_value_type_str_avoids_yaml_parsing():
439+
source = ConcurrentDeclarativeSource(
440+
source_config=_MANIFEST_WITH_VALUE_TYPE_STR, config=_CONFIG, catalog=None, state=None
441+
)
442+
443+
for stream in source.streams(_CONFIG):
444+
params = (
445+
stream._stream_partition_generator._partition_factory._retriever.requester._parameters
446+
)
447+
448+
# Confirm the usual param is still present
449+
assert "item_id" in params
450+
451+
# value_type="string" -> keep as string
452+
assert "as_string" in params
453+
assert isinstance(params["as_string"], str)
454+
assert params["as_string"] == "true"
455+
456+
assert "json_string" in params
457+
assert isinstance(params["json_string"], str)
458+
assert params["json_string"] == "[1, 2]"
459+
460+
# No value_type -> YAML parsed
461+
assert "as_yaml" in params
462+
assert isinstance(params["as_yaml"], bool)
463+
assert params["as_yaml"] is True
464+
465+
assert "json_parsed" in params
466+
assert isinstance(params["json_parsed"], list)
467+
assert params["json_parsed"] == [1, 2]
468+
469+
# value_type="string" -> remains a plain string
470+
assert "date_as_string" in params
471+
assert isinstance(params["date_as_string"], str)
472+
assert params["date_as_string"] == "2024-07-10"
473+
474+
# no value_type -> YAML parses to datetime.date
475+
assert "date_yaml_parsed" in params
476+
assert isinstance(params["date_yaml_parsed"], dt_date)
477+
assert params["date_yaml_parsed"].isoformat() == "2024-07-10"
478+

0 commit comments

Comments
 (0)