From b521ed7962740379a1c603e7f2e04066bc6f337a Mon Sep 17 00:00:00 2001 From: adamghill Date: Mon, 1 Sep 2025 22:13:54 -0400 Subject: [PATCH 1/5] Refactor `parse_file` to be more OO. Add initial support for $value and $type. --- src/dj_toml_settings/__init__.py | 4 +- src/dj_toml_settings/config.py | 5 +- src/dj_toml_settings/toml_parser.py | 326 ++++++++---------- .../value_parsers/__init__.py | 0 .../value_parsers/dict_value_parsers.py | 145 ++++++++ tests/test_toml_parser/test_parse_file.py | 106 +++--- 6 files changed, 354 insertions(+), 232 deletions(-) create mode 100644 src/dj_toml_settings/value_parsers/__init__.py create mode 100644 src/dj_toml_settings/value_parsers/dict_value_parsers.py diff --git a/src/dj_toml_settings/__init__.py b/src/dj_toml_settings/__init__.py index 4ed5a5b..5c41ddc 100644 --- a/src/dj_toml_settings/__init__.py +++ b/src/dj_toml_settings/__init__.py @@ -1,8 +1,8 @@ from dj_toml_settings.config import configure_toml_settings, get_toml_settings -from dj_toml_settings.toml_parser import parse_file +from dj_toml_settings.toml_parser import Parser __all__ = [ + "Parser", "configure_toml_settings", "get_toml_settings", - "parse_file", ] diff --git a/src/dj_toml_settings/config.py b/src/dj_toml_settings/config.py index 47a60af..2bf4390 100644 --- a/src/dj_toml_settings/config.py +++ b/src/dj_toml_settings/config.py @@ -2,7 +2,7 @@ from typeguard import typechecked -from dj_toml_settings.toml_parser import parse_file +from dj_toml_settings.toml_parser import Parser TOML_SETTINGS_FILES = ["pyproject.toml", "django.toml"] @@ -21,8 +21,9 @@ def get_toml_settings(base_dir: Path, data: dict | None = None, toml_settings_fi for settings_file_name in toml_settings_files: settings_path = base_dir / settings_file_name + if settings_path.exists(): - file_data = parse_file(settings_path, data=data.copy()) + file_data = Parser(settings_path, data=data.copy()).parse_file() data.update(file_data) return data diff --git a/src/dj_toml_settings/toml_parser.py b/src/dj_toml_settings/toml_parser.py index 10ef055..021643a 100644 --- a/src/dj_toml_settings/toml_parser.py +++ b/src/dj_toml_settings/toml_parser.py @@ -9,179 +9,20 @@ from dateutil import parser as dateparser from typeguard import typechecked -from dj_toml_settings.exceptions import InvalidActionError +from dj_toml_settings.value_parsers.dict_value_parsers import ( + EnvValueParser, + InsertValueParser, + NoneValueParser, + PathValueParser, + TypeValueParser, + ValueValueParser, +) logger = logging.getLogger(__name__) @typechecked -def parse_file(path: Path, data: dict | None = None): - """Parse data from the specified TOML file to use for Django settings. - - The sections get parsed in the following order with the later sections overriding the earlier: - 1. `[tool.django]` - 2. `[tool.django.apps.*]` - 3. `[tool.django.envs.{ENVIRONMENT}]` where {ENVIRONMENT} is defined in the `ENVIRONMENT` env variable - """ - - toml_data = get_data(path) - data = data or {} - - # Get potential settings from `tool.django.apps` and `tool.django.envs` - apps_data = toml_data.pop("apps", {}) - envs_data = toml_data.pop("envs", {}) - - # Add default settings from `tool.django` - for key, value in toml_data.items(): - logger.debug(f"tool.django: Update '{key}' with '{value}'") - - data.update(parse_key_value(data, key, value, path)) - - # Add settings from `tool.django.apps.*` - for apps_name, apps_value in apps_data.items(): - for app_key, app_value in apps_value.items(): - logger.debug(f"tool.django.apps.{apps_name}: Update '{app_key}' with '{app_value}'") - - data.update(parse_key_value(data, app_key, app_value, path)) - - # Add settings from `tool.django.envs.*` if it matches the `ENVIRONMENT` env variable - if environment_env_variable := os.getenv("ENVIRONMENT"): - for envs_name, envs_value in envs_data.items(): - if environment_env_variable == envs_name: - for env_key, env_value in envs_value.items(): - logger.debug(f"tool.django.envs.{envs_name}: Update '{env_key}' with '{env_value}'") - - data.update(parse_key_value(data, env_key, env_value, path)) - - return data - - -@typechecked -def get_data(path: Path) -> dict: - """Gets the data from the passed-in TOML file.""" - - data = {} - - try: - data = toml.load(path) - except FileNotFoundError: - logger.warning(f"Cannot find file at: {path}") - except toml.TomlDecodeError: - logger.error(f"Cannot parse TOML at: {path}") - - return data.get("tool", {}).get("django", {}) or {} - - -@typechecked -def parse_key_value(data: dict, key: str, value: Any, path: Path) -> dict: - """Handle special cases for `value`. - - Special cases: - - `dict` keys - - `$env`: retrieves an environment variable; optional `default` argument - - `$path`: converts string to a `Path`; handles relative path - - `$insert`: inserts the value to an array; optional `index` argument - - `$none`: inserts the `None` value - - variables in `str` - - `datetime` - """ - - if isinstance(value, dict): - # Defaults to "$env" and "$default" - env_special_key = _get_special_key(data, "env") - default_special_key = _get_special_key(data, "default") - - # Defaults to "$path" - path_special_key = _get_special_key(data, "path") - - # Defaults to "$insert" and "$variable" - insert_special_key = _get_special_key(data, "insert") - index_special_key = _get_special_key(data, "index") - - # Defaults to "$none" - none_special_key = _get_special_key(data, "none") - - if env_special_key in value: - default_value = value.get(default_special_key) - - value = os.getenv(value[env_special_key], default_value) - elif path_special_key in value: - file_name = value[path_special_key] - - value = _parse_path(path, file_name) - elif insert_special_key in value: - insert_data = data.get(key, []) - - # Check the existing value is an array - if not isinstance(insert_data, list): - raise InvalidActionError(f"`insert` cannot be used for value of type: {type(data[key])}") - - # Insert the data - index = value.get(index_special_key, len(insert_data)) - insert_data.insert(index, value[insert_special_key]) - - # Set the value to the new data - value = insert_data - elif none_special_key in value and value.get(none_special_key): - value = None - elif isinstance(value, str): - # Handle variable substitution - for match in re.finditer(r"\$\{\w+\}", value): - data_key = value[match.start() : match.end()][2:-1] - - if variable := data.get(data_key): - if isinstance(variable, Path): - path_str = _combine_bookends(value, match, variable) - - value = Path(path_str) - elif callable(variable): - value = variable - elif isinstance(variable, int): - value = _combine_bookends(value, match, variable) - - try: - value = int(value) - except Exception: # noqa: S110 - pass - elif isinstance(variable, float): - value = _combine_bookends(value, match, variable) - - try: - value = float(value) - except Exception: # noqa: S110 - pass - elif isinstance(variable, list): - value = variable - elif isinstance(variable, dict): - value = variable - elif isinstance(variable, datetime): - value = dateparser.isoparse(str(variable)) - else: - value = value.replace(match.string, str(variable)) - else: - logger.warning(f"Missing variable substitution {value}") - elif isinstance(value, datetime): - value = dateparser.isoparse(str(value)) - - return {key: value} - - -@typechecked -def _parse_path(path: Path, file_name: str) -> Path: - """Parse a path string relative to a base path. - - Args: - file_name: Relative or absolute file name. - path: Base path to resolve file_name against. - """ - - _path = Path(path).parent if path.is_file() else path - - return (_path / file_name).resolve() - - -@typechecked -def _combine_bookends(original: str, match: re.Match, middle: Any) -> str: +def combine_bookends(original: str, match: re.Match, middle: Any) -> str: """Get the beginning of the original string before the match, and the end of the string after the match and smush the replaced value in between them to generate a new string. @@ -196,18 +37,139 @@ def _combine_bookends(original: str, match: re.Match, middle: Any) -> str: return start + str(middle) + ending -@typechecked -def _get_special_key(data: dict, key: str) -> str: - """Gets the key for the special operator. Defaults to "$" as the prefix, and "" as the suffix. - - To change in the included TOML settings, set: - ``` - TOML_SETTINGS_SPECIAL_PREFIX = "" - TOML_SETTINGS_SPECIAL_SUFFIX = "" - ``` - """ - - prefix = data.get("TOML_SETTINGS_SPECIAL_PREFIX", "$") - suffix = data.get("TOML_SETTINGS_SPECIAL_SUFFIX", "") +class Parser: + path: Path + data: dict + + def __init__(self, path: Path, data: dict | None = None): + self.path = path + self.data = data or {} + + @typechecked + def parse_file(self): + """Parse data from the specified TOML file to use for Django settings. + + The sections get parsed in the following order with the later sections overriding the earlier: + 1. `[tool.django]` + 2. `[tool.django.apps.*]` + 3. `[tool.django.envs.{ENVIRONMENT}]` where {ENVIRONMENT} is defined in the `ENVIRONMENT` env variable + """ + + toml_data = self.get_data() + + # Get potential settings from `tool.django.apps` and `tool.django.envs` + apps_data = toml_data.pop("apps", {}) + envs_data = toml_data.pop("envs", {}) + + # Add default settings from `tool.django` + for key, value in toml_data.items(): + logger.debug(f"tool.django: Update '{key}' with '{value}'") + + self.data.update(self.parse_key_value(key, value)) + + # Add settings from `tool.django.apps.*` + for apps_name, apps_value in apps_data.items(): + for app_key, app_value in apps_value.items(): + logger.debug(f"tool.django.apps.{apps_name}: Update '{app_key}' with '{app_value}'") + + self.data.update(self.parse_key_value(app_key, app_value)) + + # Add settings from `tool.django.envs.*` if it matches the `ENVIRONMENT` env variable + if environment_env_variable := os.getenv("ENVIRONMENT"): + for envs_name, envs_value in envs_data.items(): + if environment_env_variable == envs_name: + for env_key, env_value in envs_value.items(): + logger.debug(f"tool.django.envs.{envs_name}: Update '{env_key}' with '{env_value}'") + + self.data.update(self.parse_key_value(env_key, env_value)) + + return self.data + + @typechecked + def get_data(self) -> dict: + """Gets the data from the passed-in TOML file.""" + + data = {} + + try: + data = toml.load(self.path) + except FileNotFoundError: + logger.warning(f"Cannot find file at: {self.path}") + except toml.TomlDecodeError: + logger.error(f"Cannot parse TOML at: {self.path}") + + return data.get("tool", {}).get("django", {}) or {} + + @typechecked + def parse_key_value(self, key: str, value: Any) -> dict: + """Handle special cases for `value`. + + Special cases: + - `dict` keys + - `$env`: retrieves an environment variable; optional `default` argument + - `$path`: converts string to a `Path`; handles relative path + - `$insert`: inserts the value to an array; optional `index` argument + - `$none`: inserts the `None` value + - `$value`: literal value + - `$type`: casts the value to a particular type + - variables in `str` + - `datetime` + """ + + if isinstance(value, dict): + type_parser = TypeValueParser(data=self.data, value=value) + env_parser = EnvValueParser(data=self.data, value=value) + path_parser = PathValueParser(data=self.data, value=value, path=self.path) + value_parser = ValueValueParser(data=self.data, value=value) + none_parser = NoneValueParser(data=self.data, value=value) + insert_parser = InsertValueParser(data=self.data, value=value, data_key=key) + + # Check for a match for all specials (except $type) + for parser in [env_parser, path_parser, value_parser, insert_parser, none_parser]: + if parser.match(): + value = parser.parse() + break + + # Parse $type last because it can operate on the resolved value from the other parsers + if type_parser.match(): + value = type_parser.parse(value) + elif isinstance(value, str): + # Handle variable substitution + for match in re.finditer(r"\$\{\w+\}", value): + data_key = value[match.start() : match.end()][2:-1] + + if variable := self.data.get(data_key): + if isinstance(variable, Path): + path_str = combine_bookends(value, match, variable) + + value = Path(path_str) + elif callable(variable): + value = variable + elif isinstance(variable, int): + value = combine_bookends(value, match, variable) + + try: + value = int(value) + except Exception: # noqa: S110 + pass + elif isinstance(variable, float): + value = combine_bookends(value, match, variable) + + try: + value = float(value) + except Exception: # noqa: S110 + pass + elif isinstance(variable, list): + value = variable + elif isinstance(variable, dict): + value = variable + elif isinstance(variable, datetime): + value = dateparser.isoparse(str(variable)) + else: + value = value.replace(match.string, str(variable)) + else: + logger.warning(f"Missing variable substitution {value}") + elif isinstance(value, datetime): + value = dateparser.isoparse(str(value)) - return f"{prefix}{key}{suffix}" + return {key: value} diff --git a/src/dj_toml_settings/value_parsers/__init__.py b/src/dj_toml_settings/value_parsers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/dj_toml_settings/value_parsers/dict_value_parsers.py b/src/dj_toml_settings/value_parsers/dict_value_parsers.py new file mode 100644 index 0000000..9167b41 --- /dev/null +++ b/src/dj_toml_settings/value_parsers/dict_value_parsers.py @@ -0,0 +1,145 @@ +import os +from pathlib import Path +from typing import Any + +from typeguard import typechecked + +from dj_toml_settings.exceptions import InvalidActionError + + +class DictValueParser: + data: dict + value: str + + def __init__(self, data: dict, value: str): + self.data = data + self.value = value + + if not self.key: + raise NotImplementedError("Missing key") + + self.key = self.add_prefix_and_suffix_to_key(self.key) + + def match(self) -> bool: + return self.key in self.value + + @typechecked + def add_prefix_and_suffix_to_key(self, key: str) -> str: + """Gets the key for the special operator. Defaults to "$" as the prefix, and "" as the suffix. + + To change in the included TOML settings, set: + ``` + TOML_SETTINGS_SPECIAL_PREFIX = "" + TOML_SETTINGS_SPECIAL_SUFFIX = "" + ``` + """ + + prefix = self.data.get("TOML_SETTINGS_SPECIAL_PREFIX", "$") + suffix = self.data.get("TOML_SETTINGS_SPECIAL_SUFFIX", "") + + return f"{prefix}{key}{suffix}" + + def parse(self): + raise NotImplementedError("parse() not implemented") + + +class EnvValueParser(DictValueParser): + key: str = "env" + + def parse(self) -> Any: + default_special_key = self.add_prefix_and_suffix_to_key("default") + default_value = self.value.get(default_special_key) + + env_value = self.value[self.key] + value = os.getenv(env_value, default_value) + + return value + + +class PathValueParser(DictValueParser): + key: str = "path" + + def __init__(self, data: dict, value: str, path: Path): + super().__init__(data, value) + self.path = path + + def parse(self) -> Any: + self.file_name = self.value[self.key] + value = self.resolve_file_name() + + return value + + @typechecked + def resolve_file_name(self) -> Path: + """Parse a path string relative to a base path. + + Args: + file_name: Relative or absolute file name. + path: Base path to resolve file_name against. + """ + + current_path = Path(self.path).parent if self.path.is_file() else self.path + + return (current_path / self.file_name).resolve() + + +class ValueValueParser(DictValueParser): + key = "value" + + def parse(self) -> Any: + return self.value[self.key] + + +class InsertValueParser(DictValueParser): + key = "insert" + + def __init__(self, data: dict, value: str, data_key: str): + super().__init__(data, value) + self.data_key = data_key + + def parse(self) -> Any: + insert_data = self.data.get(self.data_key, []) + + # Check the existing value is an array + if not isinstance(insert_data, list): + raise InvalidActionError(f"`insert` cannot be used for value of type: {type(self.data[self.data_key])}") + + # Insert the data + index_key = self.add_prefix_and_suffix_to_key("index") + index = self.value.get(index_key, len(insert_data)) + + insert_data.insert(index, self.value[self.key]) + + return insert_data + + +class NoneValueParser(DictValueParser): + key = "none" + + def match(self) -> bool: + return super().match() and self.value.get(self.key) + + def parse(self) -> Any: + return None + + +class TypeValueParser(DictValueParser): + key = "type" + + def parse(self, resolved_value: Any) -> Any: + value_type = self.value[self.key] + + if value_type == "bool": + if isinstance(resolved_value, str): + if resolved_value == "False": + resolved_value = False + elif resolved_value == "True": + resolved_value = True + elif isinstance(resolved_value, int): + if resolved_value == 0: + resolved_value = False + elif resolved_value == 1: + resolved_value = True + # TODO: add other types similar to environs + + return resolved_value diff --git a/tests/test_toml_parser/test_parse_file.py b/tests/test_toml_parser/test_parse_file.py index 2c992e2..c0a44b9 100644 --- a/tests/test_toml_parser/test_parse_file.py +++ b/tests/test_toml_parser/test_parse_file.py @@ -6,7 +6,7 @@ from dateutil import parser as dateparser from dj_toml_settings.exceptions import InvalidActionError -from dj_toml_settings.toml_parser import parse_file +from dj_toml_settings.toml_parser import Parser def test(tmp_path): @@ -20,7 +20,21 @@ def test(tmp_path): ] """) - actual = parse_file(path) + actual = Parser(path).parse_file() + + assert expected == actual + + +def test_type_bool_true(tmp_path): + expected = {"DEBUG": True} + + path = tmp_path / "pyproject.toml" + path.write_text(""" +[tool.django] +DEBUG = { $value = "True", $type = "bool" } +""") + + actual = Parser(path).parse_file() assert expected == actual @@ -37,7 +51,7 @@ def test_data(tmp_path): """) data = {"DEBUG": False} - actual = parse_file(path, data=data) + actual = Parser(path, data=data).parse_file() assert expected == actual @@ -52,7 +66,7 @@ def test_data_updated(tmp_path): """) data = {"DEBUG": False} - actual = parse_file(path, data=data) + actual = Parser(path, data=data).parse_file() assert expected == actual @@ -73,7 +87,7 @@ def test_apps(tmp_path): ] """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -96,7 +110,7 @@ def test_environment(tmp_path, monkeypatch): ] """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -119,7 +133,7 @@ def test_production_missing_env(tmp_path, monkeypatch): ] """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -147,7 +161,7 @@ def test_precedence(tmp_path, monkeypatch): ] """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -163,7 +177,7 @@ def test_env(tmp_path, monkeypatch): SOMETHING = { $env = "SOME_VAR" } """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -179,7 +193,7 @@ def test_env_quoted_key(tmp_path, monkeypatch): SOMETHING = { "$env" = "SOME_VAR" } """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -193,7 +207,7 @@ def test_env_missing(tmp_path): SOMETHING = { $env = "SOME_VAR" } """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -207,7 +221,7 @@ def test_env_default(tmp_path): SOMETHING = { $env = "SOME_VAR", $default = "default" } """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -221,7 +235,7 @@ def test_path(tmp_path): SOMETHING = { $path = "test-file" } """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -235,7 +249,7 @@ def test_relative_path(tmp_path): SOMETHING = { $path = "./test-file" } """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -249,7 +263,7 @@ def test_parent_path(tmp_path): SOMETHING = { $path = "../test-file" } """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -263,7 +277,7 @@ def test_parent_path_2(tmp_path): SOMETHING = { $path = "./../test-file" } """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -280,7 +294,7 @@ def test_insert(tmp_path): SOMETHING = { $insert = 2 } """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -294,7 +308,7 @@ def test_insert_missing(tmp_path): SOMETHING = { $insert = 1 } """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -312,7 +326,7 @@ def test_insert_invalid(tmp_path): """) with pytest.raises(InvalidActionError) as e: - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -334,7 +348,7 @@ def test_insert_index(tmp_path): SOMETHING = { $insert = 2, $index = 0 } """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -351,7 +365,7 @@ def test_inline_table(tmp_path): SOMETHING = { blob = "hello" } """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -365,7 +379,7 @@ def test_table(tmp_path): blob = "hello" """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -381,7 +395,7 @@ def test_all_dictionaries(tmp_path): DATABASES = { default = { ENGINE = "django.db.backends.postgresql" } } """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual # table for DATABASES @@ -390,7 +404,7 @@ def test_all_dictionaries(tmp_path): default = { ENGINE = "django.db.backends.postgresql" } """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual # table for DATABASES.default @@ -399,7 +413,7 @@ def test_all_dictionaries(tmp_path): ENGINE = "django.db.backends.postgresql" """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -415,7 +429,7 @@ def test_variable(tmp_path): SOMETHING2 = "${SOMETHING}" """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -432,7 +446,7 @@ def some_function(): SOMETHING = "${some_function}" """) - actual = parse_file(path, {"some_function": some_function}) + actual = Parser(path, data={"some_function": some_function}).parse_file() assert id(expected["SOMETHING"]) == id(actual["SOMETHING"]) @@ -447,7 +461,7 @@ def test_variable_int(tmp_path): TEST = "${INT}4" """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -462,7 +476,7 @@ def test_variable_int_with_string(tmp_path): TEST = "a${INT}" """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -477,7 +491,7 @@ def test_variable_float(tmp_path): TEST = "${FLOAT}4" """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -492,7 +506,7 @@ def test_variable_float_with_string(tmp_path): TEST = "a${FLOAT}" """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -507,7 +521,7 @@ def test_variable_array(tmp_path): TEST = "${ARRAY}" """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -522,7 +536,7 @@ def test_variable_dictionary(tmp_path): TEST = "${HASH}" """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -539,7 +553,7 @@ def test_variable_inline_table(tmp_path): TEST = "${HASH}" """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -559,7 +573,7 @@ def test_variable_datetime_utc(tmp_path): TEST = "${DATETIME}" """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -579,7 +593,7 @@ def test_variable_datetime_tz(tmp_path): TEST = "${DATETIME}" """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -593,7 +607,7 @@ def test_none(tmp_path): TEST = { $none = 1 } """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -611,7 +625,7 @@ def test_special_prefix(tmp_path): TEST = { &none = 1 } """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -631,7 +645,7 @@ def test_special_suffix(tmp_path): TEST = { none* = 1 } """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -651,7 +665,7 @@ def test_special_prefix_and_suffix(tmp_path): TEST = { &none* = 1 } """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -669,7 +683,7 @@ def test_variable_invalid(tmp_path, caplog): """) with caplog.at_level(logging.WARNING): - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -693,7 +707,7 @@ def test_variable_missing(tmp_path, caplog): """) with caplog.at_level(logging.WARNING): - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -714,7 +728,7 @@ def test_variable_start_path(tmp_path): STATIC_ROOT = "${BASE_DIR}/staticfiles" """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -729,7 +743,7 @@ def test_variable_end_path(tmp_path): STATIC_ROOT = "/blob${BASE_DIR}" """) - actual = parse_file(path) + actual = Parser(path).parse_file() assert expected == actual @@ -741,7 +755,7 @@ def test_invalid_toml(tmp_path, caplog): expected = "Cannot parse TOML at: " with caplog.at_level(logging.ERROR): - parse_file(path) + Parser(path).parse_file() # Check that an error was logged assert len(caplog.records) == 1 @@ -754,7 +768,7 @@ def test_missing_file(caplog): expected = "Cannot find file at: missing-file" with caplog.at_level(logging.WARNING): - parse_file(Path("missing-file")) + Parser(Path("missing-file")).parse_file() # Check that an error was logged assert len(caplog.records) == 1 From d9757bb3f351367e15f043e5acbbf96316864abb Mon Sep 17 00:00:00 2001 From: adamghill Date: Wed, 3 Sep 2025 07:20:53 -0400 Subject: [PATCH 2/5] Parse items in value lists. Make parser for variables. --- src/dj_toml_settings/toml_parser.py | 95 ++++++++----------- ...{dict_value_parsers.py => dict_parsers.py} | 14 +-- .../value_parsers/str_parsers.py | 75 +++++++++++++++ tests/test_toml_parser/test_parse_file.py | 48 ++++++++++ 4 files changed, 171 insertions(+), 61 deletions(-) rename src/dj_toml_settings/value_parsers/{dict_value_parsers.py => dict_parsers.py} (93%) create mode 100644 src/dj_toml_settings/value_parsers/str_parsers.py diff --git a/src/dj_toml_settings/toml_parser.py b/src/dj_toml_settings/toml_parser.py index 021643a..5d60fa5 100644 --- a/src/dj_toml_settings/toml_parser.py +++ b/src/dj_toml_settings/toml_parser.py @@ -9,14 +9,15 @@ from dateutil import parser as dateparser from typeguard import typechecked -from dj_toml_settings.value_parsers.dict_value_parsers import ( - EnvValueParser, - InsertValueParser, - NoneValueParser, - PathValueParser, - TypeValueParser, - ValueValueParser, +from dj_toml_settings.value_parsers.dict_parsers import ( + EnvParser, + InsertParser, + NoneParser, + PathParser, + TypeParser, + ValueParser, ) +from dj_toml_settings.value_parsers.str_parsers import VariableParser logger = logging.getLogger(__name__) @@ -65,14 +66,14 @@ def parse_file(self): for key, value in toml_data.items(): logger.debug(f"tool.django: Update '{key}' with '{value}'") - self.data.update(self.parse_key_value(key, value)) + self.data.update({key: self.parse_value(key, value)}) # Add settings from `tool.django.apps.*` for apps_name, apps_value in apps_data.items(): for app_key, app_value in apps_value.items(): logger.debug(f"tool.django.apps.{apps_name}: Update '{app_key}' with '{app_value}'") - self.data.update(self.parse_key_value(app_key, app_value)) + self.data.update({app_key: self.parse_value(app_key, app_value)}) # Add settings from `tool.django.envs.*` if it matches the `ENVIRONMENT` env variable if environment_env_variable := os.getenv("ENVIRONMENT"): @@ -81,7 +82,7 @@ def parse_file(self): for env_key, env_value in envs_value.items(): logger.debug(f"tool.django.envs.{envs_name}: Update '{env_key}' with '{env_value}'") - self.data.update(self.parse_key_value(env_key, env_value)) + self.data.update({env_key: self.parse_value(env_key, env_value)}) return self.data @@ -101,7 +102,7 @@ def get_data(self) -> dict: return data.get("tool", {}).get("django", {}) or {} @typechecked - def parse_key_value(self, key: str, value: Any) -> dict: + def parse_value(self, key: Any, value: Any) -> Any: """Handle special cases for `value`. Special cases: @@ -116,13 +117,33 @@ def parse_key_value(self, key: str, value: Any) -> dict: - `datetime` """ - if isinstance(value, dict): - type_parser = TypeValueParser(data=self.data, value=value) - env_parser = EnvValueParser(data=self.data, value=value) - path_parser = PathValueParser(data=self.data, value=value, path=self.path) - value_parser = ValueValueParser(data=self.data, value=value) - none_parser = NoneValueParser(data=self.data, value=value) - insert_parser = InsertValueParser(data=self.data, value=value, data_key=key) + if isinstance(value, list): + processed_list = [] + + for item in value: + processed_item = self.parse_value(key, item) + processed_list.append(processed_item) + + value = processed_list + elif isinstance(value, dict): + # Process nested dictionaries + processed_dict = {} + + for k, v in value.items(): + if isinstance(v, dict): + processed_dict.update({k: self.parse_value(key, v)}) + else: + processed_dict[k] = v + + if processed_dict: + value = processed_dict + + type_parser = TypeParser(data=self.data, value=value) + env_parser = EnvParser(data=self.data, value=value) + path_parser = PathParser(data=self.data, value=value, path=self.path) + value_parser = ValueParser(data=self.data, value=value) + none_parser = NoneParser(data=self.data, value=value) + insert_parser = InsertParser(data=self.data, value=value, data_key=key) # Check for a match for all specials (except $type) for parser in [env_parser, path_parser, value_parser, insert_parser, none_parser]: @@ -134,42 +155,8 @@ def parse_key_value(self, key: str, value: Any) -> dict: if type_parser.match(): value = type_parser.parse(value) elif isinstance(value, str): - # Handle variable substitution - for match in re.finditer(r"\$\{\w+\}", value): - data_key = value[match.start() : match.end()][2:-1] - - if variable := self.data.get(data_key): - if isinstance(variable, Path): - path_str = combine_bookends(value, match, variable) - - value = Path(path_str) - elif callable(variable): - value = variable - elif isinstance(variable, int): - value = combine_bookends(value, match, variable) - - try: - value = int(value) - except Exception: # noqa: S110 - pass - elif isinstance(variable, float): - value = combine_bookends(value, match, variable) - - try: - value = float(value) - except Exception: # noqa: S110 - pass - elif isinstance(variable, list): - value = variable - elif isinstance(variable, dict): - value = variable - elif isinstance(variable, datetime): - value = dateparser.isoparse(str(variable)) - else: - value = value.replace(match.string, str(variable)) - else: - logger.warning(f"Missing variable substitution {value}") + value = VariableParser(data=self.data, value=value).parse() elif isinstance(value, datetime): value = dateparser.isoparse(str(value)) - return {key: value} + return value diff --git a/src/dj_toml_settings/value_parsers/dict_value_parsers.py b/src/dj_toml_settings/value_parsers/dict_parsers.py similarity index 93% rename from src/dj_toml_settings/value_parsers/dict_value_parsers.py rename to src/dj_toml_settings/value_parsers/dict_parsers.py index 9167b41..696d7db 100644 --- a/src/dj_toml_settings/value_parsers/dict_value_parsers.py +++ b/src/dj_toml_settings/value_parsers/dict_parsers.py @@ -7,7 +7,7 @@ from dj_toml_settings.exceptions import InvalidActionError -class DictValueParser: +class DictParser: data: dict value: str @@ -43,7 +43,7 @@ def parse(self): raise NotImplementedError("parse() not implemented") -class EnvValueParser(DictValueParser): +class EnvParser(DictParser): key: str = "env" def parse(self) -> Any: @@ -56,7 +56,7 @@ def parse(self) -> Any: return value -class PathValueParser(DictValueParser): +class PathParser(DictParser): key: str = "path" def __init__(self, data: dict, value: str, path: Path): @@ -83,14 +83,14 @@ def resolve_file_name(self) -> Path: return (current_path / self.file_name).resolve() -class ValueValueParser(DictValueParser): +class ValueParser(DictParser): key = "value" def parse(self) -> Any: return self.value[self.key] -class InsertValueParser(DictValueParser): +class InsertParser(DictParser): key = "insert" def __init__(self, data: dict, value: str, data_key: str): @@ -113,7 +113,7 @@ def parse(self) -> Any: return insert_data -class NoneValueParser(DictValueParser): +class NoneParser(DictParser): key = "none" def match(self) -> bool: @@ -123,7 +123,7 @@ def parse(self) -> Any: return None -class TypeValueParser(DictValueParser): +class TypeParser(DictParser): key = "type" def parse(self, resolved_value: Any) -> Any: diff --git a/src/dj_toml_settings/value_parsers/str_parsers.py b/src/dj_toml_settings/value_parsers/str_parsers.py new file mode 100644 index 0000000..a4c19b9 --- /dev/null +++ b/src/dj_toml_settings/value_parsers/str_parsers.py @@ -0,0 +1,75 @@ +import logging +import re +from datetime import datetime +from pathlib import Path +from typing import Any + +from dateutil import parser as dateparser +from typeguard import typechecked + +logger = logging.getLogger(__name__) + + +@typechecked +def combine_bookends(original: str, match: re.Match, middle: Any) -> str: + """Get the beginning of the original string before the match, and the + end of the string after the match and smush the replaced value in between + them to generate a new string. + """ + + start_idx = match.start() + start = original[:start_idx] + + end_idx = match.end() + ending = original[end_idx:] + + return start + str(middle) + ending + + +class VariableParser: + data: dict + value: str + + def __init__(self, data: dict, value: str): + self.data = data + self.value = value + + def parse(self) -> Any: + value = self.value + + for match in re.finditer(r"\$\{\w+\}", value): + data_key = value[match.start() : match.end()][2:-1] + + if variable := self.data.get(data_key): + if isinstance(variable, Path): + path_str = combine_bookends(value, match, variable) + + value = Path(path_str) + elif callable(variable): + value = variable + elif isinstance(variable, int): + value = combine_bookends(value, match, variable) + + try: + value = int(value) + except Exception: # noqa: S110 + pass + elif isinstance(variable, float): + value = combine_bookends(value, match, variable) + + try: + value = float(value) + except Exception: # noqa: S110 + pass + elif isinstance(variable, list): + value = variable + elif isinstance(variable, dict): + value = variable + elif isinstance(variable, datetime): + value = dateparser.isoparse(str(variable)) + else: + value = value.replace(match.string, str(variable)) + else: + logger.warning(f"Missing variable substitution {value}") + + return value diff --git a/tests/test_toml_parser/test_parse_file.py b/tests/test_toml_parser/test_parse_file.py index c0a44b9..3cc799a 100644 --- a/tests/test_toml_parser/test_parse_file.py +++ b/tests/test_toml_parser/test_parse_file.py @@ -182,6 +182,54 @@ def test_env(tmp_path, monkeypatch): assert expected == actual +def test_env_in_nested_dict(tmp_path, monkeypatch): + monkeypatch.setenv("SOME_VAR", "blob") + + expected = {"SOMETHING": {"MORE": "blob"}} + + path = tmp_path / "pyproject.toml" + path.write_text(""" +[tool.django] +SOMETHING = { MORE = { $env = "SOME_VAR" } } +""") + + actual = Parser(path).parse_file() + + assert expected == actual + + +def test_env_in_list(tmp_path, monkeypatch): + monkeypatch.setenv("SOME_VAR", "blob") + + expected = {"SOMETHING": ["blob"]} + + path = tmp_path / "pyproject.toml" + path.write_text(""" +[tool.django] +SOMETHING = [{ "$env" = "SOME_VAR"}] +""") + + actual = Parser(path).parse_file() + + assert expected == actual + + +def test_variable_in_list(tmp_path): + expected = {"SOMETHING": ["blob"], "SOME_VAR": "blob"} + + path = tmp_path / "pyproject.toml" + path.write_text(""" +[tool.django] +SOME_VAR = "blob" +SOMETHING = ["${SOME_VAR}"] +""") + + actual = Parser(path).parse_file() + print(actual) + + assert expected == actual + + def test_env_quoted_key(tmp_path, monkeypatch): monkeypatch.setenv("SOME_VAR", "blob") From af68d4418e56c60f3d36e1a2bb5bf6c0ea3aff54 Mon Sep 17 00:00:00 2001 From: adamghill Date: Thu, 4 Sep 2025 07:54:53 -0400 Subject: [PATCH 3/5] Add support for float, int, str, decimal, datetime, date, time, timedelta, url to $type. --- src/dj_toml_settings/toml_parser.py | 20 +-- .../value_parsers/dict_parsers.py | 111 +++++++++++-- .../value_parsers/str_parsers.py | 32 ++-- tests/test_toml_parser/test_parse_file.py | 14 ++ .../test_dict_parsers/test_type_parser.py | 154 ++++++++++++++++++ 5 files changed, 283 insertions(+), 48 deletions(-) create mode 100644 tests/test_value_parsers/test_dict_parsers/test_type_parser.py diff --git a/src/dj_toml_settings/toml_parser.py b/src/dj_toml_settings/toml_parser.py index 5d60fa5..2b2479c 100644 --- a/src/dj_toml_settings/toml_parser.py +++ b/src/dj_toml_settings/toml_parser.py @@ -1,6 +1,5 @@ import logging import os -import re from datetime import datetime from pathlib import Path from typing import Any @@ -22,22 +21,6 @@ logger = logging.getLogger(__name__) -@typechecked -def combine_bookends(original: str, match: re.Match, middle: Any) -> str: - """Get the beginning of the original string before the match, and the - end of the string after the match and smush the replaced value in between - them to generate a new string. - """ - - start_idx = match.start() - start = original[:start_idx] - - end_idx = match.end() - ending = original[end_idx:] - - return start + str(middle) + ending - - class Parser: path: Path data: dict @@ -118,6 +101,7 @@ def parse_value(self, key: Any, value: Any) -> Any: """ if isinstance(value, list): + # Process each item in the list processed_list = [] for item in value: @@ -145,7 +129,7 @@ def parse_value(self, key: Any, value: Any) -> Any: none_parser = NoneParser(data=self.data, value=value) insert_parser = InsertParser(data=self.data, value=value, data_key=key) - # Check for a match for all specials (except $type) + # Check for a match for all operators (except $type) for parser in [env_parser, path_parser, value_parser, insert_parser, none_parser]: if parser.match(): value = parser.parse() diff --git a/src/dj_toml_settings/value_parsers/dict_parsers.py b/src/dj_toml_settings/value_parsers/dict_parsers.py index 696d7db..86a3a02 100644 --- a/src/dj_toml_settings/value_parsers/dict_parsers.py +++ b/src/dj_toml_settings/value_parsers/dict_parsers.py @@ -1,11 +1,20 @@ +import json +import logging import os +import re +from datetime import timedelta +from decimal import Decimal from pathlib import Path from typing import Any +from urllib.parse import urlparse +from dateutil import parser as dateparser from typeguard import typechecked from dj_toml_settings.exceptions import InvalidActionError +logger = logging.getLogger(__name__) + class DictParser: data: dict @@ -129,17 +138,91 @@ class TypeParser(DictParser): def parse(self, resolved_value: Any) -> Any: value_type = self.value[self.key] - if value_type == "bool": - if isinstance(resolved_value, str): - if resolved_value == "False": - resolved_value = False - elif resolved_value == "True": - resolved_value = True - elif isinstance(resolved_value, int): - if resolved_value == 0: - resolved_value = False - elif resolved_value == 1: - resolved_value = True - # TODO: add other types similar to environs - - return resolved_value + if not isinstance(value_type, str): + raise ValueError(f"Type must be a string, got {type(value_type).__name__}") + + try: + if value_type == "bool": + if isinstance(resolved_value, str): + resolved_value = resolved_value.lower() == "true" + elif isinstance(resolved_value, int): + resolved_value = bool(resolved_value) + return bool(resolved_value) + elif value_type == "int": + return int(resolved_value) + elif value_type == "str": + return str(resolved_value) + elif value_type == "float": + return float(resolved_value) + elif value_type == "decimal": + return Decimal(str(resolved_value)) + elif value_type == "datetime": + result = dateparser.parse(resolved_value) + + if not result: + raise ValueError(f"Could not parse datetime from: {resolved_value}") + + return result + elif value_type == "date": + result = dateparser.parse(resolved_value) + + if not result: + raise ValueError(f"Could not parse date from: {resolved_value}") + + return result.date() + elif value_type == "time": + result = dateparser.parse(resolved_value) + + if not result: + raise ValueError(f"Could not parse time from: {resolved_value}") + + return result.time() + elif value_type == "timedelta": + return parse_timedelta(resolved_value) + elif value_type == "url": + return urlparse(str(resolved_value)) + else: + raise ValueError(f"Unsupported type: {value_type}") + except (ValueError, TypeError, AttributeError) as e: + logger.debug(f"Failed to convert {resolved_value!r} to {value_type}: {e}") + + raise ValueError(f"Failed to convert {resolved_value!r} to {value_type}: {e}") from e + + +def parse_timedelta(value): + if isinstance(value, int | float): + return timedelta(seconds=value) + elif not isinstance(value, str): + raise ValueError(f"Unsupported type for timedelta: {type(value).__name__}") + + # Pattern to match both space-separated and combined formats like '7w2d' + pattern = r"(?:\s*(\d+\.?\d*)([a-z]+))" + matches = re.findall(pattern, value, re.IGNORECASE) + + if not matches and value.strip(): + raise ValueError(f"Invalid timedelta format: {value}") + + unit_map = { + "u": "microseconds", + "ms": "milliseconds", + "s": "seconds", + "m": "minutes", + "h": "hours", + "d": "days", + "w": "weeks", + } + kwargs = {} + + for num_str, unit in matches: + try: + num = float(num_str) + except ValueError: + raise ValueError(f"Invalid number in timedelta: {num_str}") + + if unit not in unit_map: + raise ValueError(f"Invalid time unit: {unit}") + + key = unit_map[unit] + kwargs[key] = kwargs.get(key, 0) + num + + return timedelta(**kwargs) diff --git a/src/dj_toml_settings/value_parsers/str_parsers.py b/src/dj_toml_settings/value_parsers/str_parsers.py index a4c19b9..c7cfbca 100644 --- a/src/dj_toml_settings/value_parsers/str_parsers.py +++ b/src/dj_toml_settings/value_parsers/str_parsers.py @@ -10,22 +10,6 @@ logger = logging.getLogger(__name__) -@typechecked -def combine_bookends(original: str, match: re.Match, middle: Any) -> str: - """Get the beginning of the original string before the match, and the - end of the string after the match and smush the replaced value in between - them to generate a new string. - """ - - start_idx = match.start() - start = original[:start_idx] - - end_idx = match.end() - ending = original[end_idx:] - - return start + str(middle) + ending - - class VariableParser: data: dict value: str @@ -73,3 +57,19 @@ def parse(self) -> Any: logger.warning(f"Missing variable substitution {value}") return value + + +@typechecked +def combine_bookends(original: str, match: re.Match, middle: Any) -> str: + """Get the beginning of the original string before the match, and the + end of the string after the match and smush the replaced value in between + them to generate a new string. + """ + + start_idx = match.start() + start = original[:start_idx] + + end_idx = match.end() + ending = original[end_idx:] + + return start + str(middle) + ending diff --git a/tests/test_toml_parser/test_parse_file.py b/tests/test_toml_parser/test_parse_file.py index 3cc799a..973fc9f 100644 --- a/tests/test_toml_parser/test_parse_file.py +++ b/tests/test_toml_parser/test_parse_file.py @@ -39,6 +39,20 @@ def test_type_bool_true(tmp_path): assert expected == actual +def test_type_float(tmp_path): + expected = {"FLOAT": float("1.5")} + + path = tmp_path / "pyproject.toml" + path.write_text(""" +[tool.django] +FLOAT = { $value = "1.5", $type = "float" } +""") + + actual = Parser(path).parse_file() + + assert expected == actual + + def test_data(tmp_path): expected = {"DEBUG": False, "ALLOWED_HOSTS": ["127.0.0.1"]} diff --git a/tests/test_value_parsers/test_dict_parsers/test_type_parser.py b/tests/test_value_parsers/test_dict_parsers/test_type_parser.py new file mode 100644 index 0000000..8819c85 --- /dev/null +++ b/tests/test_value_parsers/test_dict_parsers/test_type_parser.py @@ -0,0 +1,154 @@ +from datetime import date, datetime, time, timedelta +from decimal import Decimal +from urllib.parse import ParseResult + +import pytest + +from dj_toml_settings.value_parsers.dict_parsers import TypeParser + + +def test_bool(): + parser = TypeParser(data={}, value={"$type": "bool"}) + + # Test string conversions + assert parser.parse("true") is True + assert parser.parse("True") is True + assert parser.parse("false") is False + assert parser.parse("False") is False + + # Test integer conversions + assert parser.parse(1) is True + assert parser.parse(0) is False + + # Test already boolean + assert parser.parse(True) is True + assert parser.parse(False) is False + + +def test_int(): + parser = TypeParser(data={}, value={"$type": "int"}) + + # Test string to int + assert parser.parse("42") == 42 + assert parser.parse("-10") == -10 + + # Test float to int (should truncate) + assert parser.parse(3.14) == 3 + + # Test already int + assert parser.parse(100) == 100 + + +def test_str(): + parser = TypeParser(data={}, value={"$type": "str"}) + + # Test various types to string + assert parser.parse(42) == "42" + assert parser.parse(3.14) == "3.14" + assert parser.parse(True) == "True" + assert parser.parse(None) == "None" + assert parser.parse("hello") == "hello" + + +def test_float(): + parser = TypeParser(data={}, value={"$type": "float"}) + + # Test string to float + assert parser.parse("3.14") == 3.14 + assert parser.parse("-1.5") == -1.5 + + # Test int to float + assert parser.parse(42) == 42.0 + + # Test already float + assert parser.parse(3.14) == 3.14 + + +def test_decimal(): + parser = TypeParser(data={}, value={"$type": "decimal"}) + + # Test string to Decimal + assert parser.parse("3.14") == Decimal("3.14") + assert parser.parse("-1.5") == Decimal("-1.5") + + # Test int to Decimal + assert parser.parse(42) == Decimal(42) + + # Test float to Decimal (note: float imprecision might occur) + assert float(parser.parse(3.14)) == 3.14 + + +def test_datetime(): + parser = TypeParser(data={}, value={"$type": "datetime"}) + + # Test string to datetime + dt = parser.parse("2023-01-01 12:00:00") + assert isinstance(dt, datetime) + assert dt.year == 2023 + assert dt.month == 1 + assert dt.day == 1 + assert dt.hour == 12 + + +def test_date(): + parser = TypeParser(data={}, value={"$type": "date"}) + + # Test string to date + date_obj = parser.parse("2023-01-01") + assert isinstance(date_obj, date) + assert date_obj.year == 2023 + assert date_obj.month == 1 + assert date_obj.day == 1 + + +def test_time(): + parser = TypeParser(data={}, value={"$type": "time"}) + + # Test string to time + time_obj = parser.parse("12:30:45") + assert isinstance(time_obj, time) + assert time_obj.hour == 12 + assert time_obj.minute == 30 + assert time_obj.second == 45 + + +def test_timedelta(): + parser = TypeParser(data={}, value={"$type": "timedelta"}) + + # Test numeric values (treated as seconds) + assert parser.parse(60) == timedelta(seconds=60) + assert parser.parse(90.5) == timedelta(seconds=90.5) + + # Test string formats + assert parser.parse("30s") == timedelta(seconds=30) + assert parser.parse("2m") == timedelta(minutes=2) + assert parser.parse("1.5h") == timedelta(hours=1.5) + assert parser.parse("2d") == timedelta(days=2) + assert parser.parse("1w") == timedelta(weeks=1) + + assert parser.parse("1w 2d 3h 4m 5s 6ms 7u") == timedelta( + weeks=1, days=2, hours=3, minutes=4, seconds=5, milliseconds=6, microseconds=7 + ) + + assert parser.parse("1w2d") == timedelta(weeks=1, days=2) + + +def test_url(): + parser = TypeParser(data={}, value={"$type": "url"}) + + # Test URL parsing + result = parser.parse("https://example.com/path?query=1") + assert isinstance(result, ParseResult) + assert result.scheme == "https" + assert result.netloc == "example.com" + assert result.path == "/path" + assert result.query == "query=1" + + +def test_invalid_type(): + parser = TypeParser(data={}, value={"$type": "invalid_type"}) + + with pytest.raises(ValueError) as exc_info: + parser.parse("some value") + + assert "Unsupported type: invalid_type" in str(exc_info.value) From a974d559728c6bc1762b498c323e37672dbea696 Mon Sep 17 00:00:00 2001 From: adamghill Date: Mon, 8 Sep 2025 15:47:23 -0500 Subject: [PATCH 4/5] Fix type issues. --- .../value_parsers/dict_parsers.py | 20 +++++++++---------- .../value_parsers/str_parsers.py | 2 +- tests/test_toml_parser/test_parse_file.py | 1 - 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/dj_toml_settings/value_parsers/dict_parsers.py b/src/dj_toml_settings/value_parsers/dict_parsers.py index 86a3a02..cd04df9 100644 --- a/src/dj_toml_settings/value_parsers/dict_parsers.py +++ b/src/dj_toml_settings/value_parsers/dict_parsers.py @@ -1,4 +1,3 @@ -import json import logging import os import re @@ -18,9 +17,10 @@ class DictParser: data: dict - value: str + value: dict + key: str - def __init__(self, data: dict, value: str): + def __init__(self, data: dict, value: dict): self.data = data self.value = value @@ -48,7 +48,7 @@ def add_prefix_and_suffix_to_key(self, key: str) -> str: return f"{prefix}{key}{suffix}" - def parse(self): + def parse(self, *args, **kwargs): raise NotImplementedError("parse() not implemented") @@ -68,7 +68,7 @@ def parse(self) -> Any: class PathParser(DictParser): key: str = "path" - def __init__(self, data: dict, value: str, path: Path): + def __init__(self, data: dict, value: dict, path: Path): super().__init__(data, value) self.path = path @@ -89,7 +89,7 @@ def resolve_file_name(self) -> Path: current_path = Path(self.path).parent if self.path.is_file() else self.path - return (current_path / self.file_name).resolve() + return Path((current_path / self.file_name).resolve()) class ValueParser(DictParser): @@ -102,7 +102,7 @@ def parse(self) -> Any: class InsertParser(DictParser): key = "insert" - def __init__(self, data: dict, value: str, data_key: str): + def __init__(self, data: dict, value: dict, data_key: str): super().__init__(data, value) self.data_key = data_key @@ -126,7 +126,7 @@ class NoneParser(DictParser): key = "none" def match(self) -> bool: - return super().match() and self.value.get(self.key) + return super().match() and self.value.get(self.key) is not None def parse(self) -> Any: return None @@ -216,8 +216,8 @@ def parse_timedelta(value): for num_str, unit in matches: try: num = float(num_str) - except ValueError: - raise ValueError(f"Invalid number in timedelta: {num_str}") + except ValueError as e: + raise ValueError(f"Invalid number in timedelta: {num_str}") from e if unit not in unit_map: raise ValueError(f"Invalid time unit: {unit}") diff --git a/src/dj_toml_settings/value_parsers/str_parsers.py b/src/dj_toml_settings/value_parsers/str_parsers.py index c7cfbca..c99057b 100644 --- a/src/dj_toml_settings/value_parsers/str_parsers.py +++ b/src/dj_toml_settings/value_parsers/str_parsers.py @@ -19,7 +19,7 @@ def __init__(self, data: dict, value: str): self.value = value def parse(self) -> Any: - value = self.value + value: Any = self.value for match in re.finditer(r"\$\{\w+\}", value): data_key = value[match.start() : match.end()][2:-1] diff --git a/tests/test_toml_parser/test_parse_file.py b/tests/test_toml_parser/test_parse_file.py index 973fc9f..336835f 100644 --- a/tests/test_toml_parser/test_parse_file.py +++ b/tests/test_toml_parser/test_parse_file.py @@ -239,7 +239,6 @@ def test_variable_in_list(tmp_path): """) actual = Parser(path).parse_file() - print(actual) assert expected == actual From b3a6dc560b92ee316d8a8a6ae2fc71f3c22627b9 Mon Sep 17 00:00:00 2001 From: adamghill Date: Tue, 9 Sep 2025 07:06:24 -0500 Subject: [PATCH 5/5] Add some tests. --- src/dj_toml_settings/toml_parser.py | 3 +- .../value_parsers/dict_parsers.py | 22 +++----- .../test_dict_parsers/test_dict_parser.py | 25 +++++++++ .../test_dict_parsers/test_type_parser.py | 55 ++++++++++++++++--- 4 files changed, 79 insertions(+), 26 deletions(-) create mode 100644 tests/test_value_parsers/test_dict_parsers/test_dict_parser.py diff --git a/src/dj_toml_settings/toml_parser.py b/src/dj_toml_settings/toml_parser.py index 2b2479c..fc9bd1c 100644 --- a/src/dj_toml_settings/toml_parser.py +++ b/src/dj_toml_settings/toml_parser.py @@ -119,8 +119,7 @@ def parse_value(self, key: Any, value: Any) -> Any: else: processed_dict[k] = v - if processed_dict: - value = processed_dict + value = processed_dict type_parser = TypeParser(data=self.data, value=value) env_parser = EnvParser(data=self.data, value=value) diff --git a/src/dj_toml_settings/value_parsers/dict_parsers.py b/src/dj_toml_settings/value_parsers/dict_parsers.py index cd04df9..aa71606 100644 --- a/src/dj_toml_settings/value_parsers/dict_parsers.py +++ b/src/dj_toml_settings/value_parsers/dict_parsers.py @@ -24,8 +24,8 @@ def __init__(self, data: dict, value: dict): self.data = data self.value = value - if not self.key: - raise NotImplementedError("Missing key") + if not hasattr(self, "key"): + raise NotImplementedError("Missing key attribute") self.key = self.add_prefix_and_suffix_to_key(self.key) @@ -147,6 +147,9 @@ def parse(self, resolved_value: Any) -> Any: resolved_value = resolved_value.lower() == "true" elif isinstance(resolved_value, int): resolved_value = bool(resolved_value) + else: + raise ValueError(f"Type must be a string or int, got {type(resolved_value).__name__}") + return bool(resolved_value) elif value_type == "int": return int(resolved_value) @@ -157,25 +160,14 @@ def parse(self, resolved_value: Any) -> Any: elif value_type == "decimal": return Decimal(str(resolved_value)) elif value_type == "datetime": - result = dateparser.parse(resolved_value) - - if not result: - raise ValueError(f"Could not parse datetime from: {resolved_value}") - - return result + return dateparser.parse(resolved_value) elif value_type == "date": result = dateparser.parse(resolved_value) - if not result: - raise ValueError(f"Could not parse date from: {resolved_value}") - return result.date() elif value_type == "time": result = dateparser.parse(resolved_value) - if not result: - raise ValueError(f"Could not parse time from: {resolved_value}") - return result.time() elif value_type == "timedelta": return parse_timedelta(resolved_value) @@ -196,7 +188,7 @@ def parse_timedelta(value): raise ValueError(f"Unsupported type for timedelta: {type(value).__name__}") # Pattern to match both space-separated and combined formats like '7w2d' - pattern = r"(?:\s*(\d+\.?\d*)([a-z]+))" + pattern = r"(?:\s*(\d+\.?\d*)([u|ms|s|m|h|d|w]+))" matches = re.findall(pattern, value, re.IGNORECASE) if not matches and value.strip(): diff --git a/tests/test_value_parsers/test_dict_parsers/test_dict_parser.py b/tests/test_value_parsers/test_dict_parsers/test_dict_parser.py new file mode 100644 index 0000000..5ff92df --- /dev/null +++ b/tests/test_value_parsers/test_dict_parsers/test_dict_parser.py @@ -0,0 +1,25 @@ +import pytest + +from dj_toml_settings.value_parsers.dict_parsers import DictParser + + +class FakeParser(DictParser): + pass + + +class FakeParserWithKey(DictParser): + key = "test" + + +def test_missing_key(): + with pytest.raises(NotImplementedError) as e: + FakeParser({}, {}) + + assert "Missing key attribute" in e.exconly() + + +def test_missing_parse(): + with pytest.raises(NotImplementedError) as e: + FakeParserWithKey({}, {}).parse() + + assert "parse() not implemented" in e.exconly() diff --git a/tests/test_value_parsers/test_dict_parsers/test_type_parser.py b/tests/test_value_parsers/test_dict_parsers/test_type_parser.py index 8819c85..ad8f65a 100644 --- a/tests/test_value_parsers/test_dict_parsers/test_type_parser.py +++ b/tests/test_value_parsers/test_dict_parsers/test_type_parser.py @@ -24,6 +24,11 @@ def test_bool(): assert parser.parse(True) is True assert parser.parse(False) is False + with pytest.raises(ValueError) as e: + assert parser.parse(1.1) + + assert "ValueError: Failed to convert 1.1 to bool: Type must be a string or int, got float" in e.exconly() + def test_int(): parser = TypeParser(data={}, value={"$type": "int"}) @@ -81,13 +86,21 @@ def test_decimal(): def test_datetime(): parser = TypeParser(data={}, value={"$type": "datetime"}) - # Test string to datetime - dt = parser.parse("2023-01-01 12:00:00") - assert isinstance(dt, datetime) - assert dt.year == 2023 - assert dt.month == 1 - assert dt.day == 1 - assert dt.hour == 12 + actual = parser.parse("2023-01-01 12:00:00") + assert isinstance(actual, datetime) + assert actual.year == 2023 + assert actual.month == 1 + assert actual.day == 1 + assert actual.hour == 12 + + +def test_datetime_invalid(): + parser = TypeParser(data={}, value={"$type": "datetime"}) + + with pytest.raises(ValueError) as e: + parser.parse("abcd") + + assert "Failed to convert 'abcd' to datetime" in e.exconly() def test_date(): @@ -132,6 +145,21 @@ def test_timedelta(): assert parser.parse("1w2d") == timedelta(weeks=1, days=2) + with pytest.raises(ValueError) as e: + parser.parse({}) + + assert "ValueError: Failed to convert {} to timedelta: Unsupported type for timedelta: dict" in e.exconly() + + with pytest.raises(ValueError) as e: + parser.parse("abcd") + + assert "ValueError: Failed to convert 'abcd' to timedelta: Invalid timedelta format: abcd" in e.exconly() + + with pytest.raises(ValueError) as e: + parser.parse("4z") + + assert "alueError: Failed to convert '4z' to timedelta: Invalid timedelta format: 4z" in e.exconly() + def test_url(): parser = TypeParser(data={}, value={"$type": "url"}) @@ -148,7 +176,16 @@ def test_url(): def test_invalid_type(): parser = TypeParser(data={}, value={"$type": "invalid_type"}) - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError) as e: + parser.parse("some value") + + assert "Unsupported type: invalid_type" in e.exconly() + + +def test_invalid_type_type(): + parser = TypeParser(data={}, value={"$type": 1}) + + with pytest.raises(ValueError) as e: parser.parse("some value") - assert "Unsupported type: invalid_type" in str(exc_info.value) + assert "Type must be a string, got int" in e.exconly()