diff --git a/CHANGES.md b/CHANGES.md index 7a94c4e..eb73348 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,15 @@ ## Version 0.2.0 (in development) - Make all docstrings comply to google-style +- Rule description is now your `RuleOp`'s docstring + if `description` is not explicitly provided. +- Supporting _virtual plugins_: plugins provided by Python + dictionaries with rules defined by the `RuleOp` classes. +- Added more configuration examples in the `examples` folder. +- Introduced utilities `ValueConstructible` and + derived `MappingConstructible` which greatly simplify + flexible instantiation of configuration objects and their + children from Python and JSON/YAML values. ## Version 0.1.0 (09.01.2025) diff --git a/docs/api.md b/docs/api.md index c1204c0..cfc9fa6 100644 --- a/docs/api.md +++ b/docs/api.md @@ -16,20 +16,23 @@ This chapter provides a plain reference for the XRLint Python API. - The `config` module provides classes that represent configuration information and provide related functionality: [Config][xrlint.config.Config] and [ConfigList][xrlint.config.ConfigList]. -- The `rule` module provides rule related classes: +- The `rule` module provides rule related classes and functions: [Rule][xrlint.rule.Rule] comprising rule metadata [RuleMeta][xrlint.rule.RuleMeta] and the rule operation [RuleOp][xrlint.rule.RuleOp], as well as related to the latter [RuleContext][xrlint.rule.RuleContext] and [RuleExit][xrlint.rule.RuleExit]. + Decorator [define_rule][xrlint.rule.define_rule] allows defining rules. - The `node` module defines the nodes passed to [xrlint.rule.RuleOp]: base classes [None][xrlint.node.Node], [XarrayNode][xrlint.node.XarrayNode] and the specific [DatasetNode][xrlint.node.DatasetNode], [DataArray][xrlint.node.DataArrayNode], [AttrsNode][xrlint.node.AttrsNode], and [AttrNode][xrlint.node.AttrNode] nodes. -- The `processor` module provides processor related classes: +- The `processor` module provides processor related classes and functions: [Processor][xrlint.processor.Processor] comprising processor metadata [ProcessorMeta][xrlint.processor.ProcessorMeta] and the processor operation [ProcessorOp][xrlint.processor.ProcessorOp]. + Decorator [define_processor][xrlint.processor.define_processor] allows defining + processors. - The `result` module provides data classes that are used to represent validation results: [Result][xrlint.result.Result] composed of [Messages][xrlint.result.Message], @@ -51,6 +54,8 @@ Note: the `xrlint.all` convenience module exports all of the above from a ::: xrlint.config.ConfigList +::: xrlint.rule.define_rule + ::: xrlint.rule.Rule ::: xrlint.rule.RuleMeta @@ -77,6 +82,8 @@ Note: the `xrlint.all` convenience module exports all of the above from a ::: xrlint.plugin.PluginMeta +::: xrlint.processor.define_processor + ::: xrlint.processor.Processor ::: xrlint.processor.ProcessorMeta diff --git a/environment.yml b/environment.yml index faf822f..b043546 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,4 @@ -name: xrlint +name: xrlint-310 channels: - conda-forge dependencies: diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/xrlint_config.py b/examples/plugin_config.py similarity index 61% rename from examples/xrlint_config.py rename to examples/plugin_config.py index 9f8971e..7487fbf 100644 --- a/examples/xrlint_config.py +++ b/examples/plugin_config.py @@ -1,4 +1,7 @@ -# XRLint configuration file example +""" +This configuration example shows how to define and use a plugin +using the `Plugin` class and its `define_rule()` decorator method. +""" from xrlint.config import Config from xrlint.node import DatasetNode @@ -11,21 +14,24 @@ plugin = Plugin( meta=PluginMeta(name="hello-plugin", version="1.0.0"), configs={ + # "configs" entries must be `Config` objects! "recommended": Config.from_value( { "rules": { - "hello/good-title": "error", + "hello/good-title": "warn", + # Configure more rules here... }, } - ) + ), + # Add more configurations here... }, ) -@plugin.define_rule( - "good-title", description=f"Dataset title should be 'Hello World!'." -) +@plugin.define_rule("good-title") class GoodTitle(RuleOp): + """Dataset title should be 'Hello World!'.""" + def dataset(self, ctx: RuleContext, node: DatasetNode): good_title = "Hello World!" if node.dataset.attrs.get("title") != good_title: @@ -35,16 +41,19 @@ def dataset(self, ctx: RuleContext, node: DatasetNode): ) +# Define more rules here... + + def export_configs(): return [ - { - "files": ["**/*.zarr", "**/*.nc"], - }, + # Use "hello" plugin { "plugins": { "hello": plugin, }, }, + # Use recommended settings from xrlint "recommended", + # Use recommended settings from "hello" plugin "hello/recommended", ] diff --git a/examples/rule_testing.py b/examples/rule_testing.py new file mode 100644 index 0000000..7ec7917 --- /dev/null +++ b/examples/rule_testing.py @@ -0,0 +1,51 @@ +""" +This example demonstrates how to develop new rules. +""" + +import xarray as xr + +from xrlint.node import DatasetNode +from xrlint.rule import RuleContext +from xrlint.rule import RuleOp +from xrlint.rule import define_rule +from xrlint.testing import RuleTest +from xrlint.testing import RuleTester + + +@define_rule("good-title") +class GoodTitle(RuleOp): + """Dataset title should be 'Hello World!'.""" + + def dataset(self, ctx: RuleContext, node: DatasetNode): + good_title = "Hello World!" + if node.dataset.attrs.get("title") != good_title: + ctx.report( + "Attribute 'title' wrong.", + suggestions=[f"Rename it to {good_title!r}."], + ) + + +# ----------------- +# In another module +# ----------------- + +tester = RuleTester() + +valid_dataset = xr.Dataset(attrs=dict(title="Hello World!")) +invalid_dataset = xr.Dataset(attrs=dict(title="Hello Hamburg!")) + +# Run test directly +tester.run( + "good-title", + GoodTitle, + valid=[RuleTest(dataset=valid_dataset)], + invalid=[RuleTest(dataset=invalid_dataset)], +) + +# or define a test class derived from unitest.TestCase +GoodTitleTest = tester.define_test( + "good-title", + GoodTitle, + valid=[RuleTest(dataset=valid_dataset)], + invalid=[RuleTest(dataset=invalid_dataset)], +) diff --git a/examples/virtual_plugin_config.py b/examples/virtual_plugin_config.py new file mode 100644 index 0000000..4837d59 --- /dev/null +++ b/examples/virtual_plugin_config.py @@ -0,0 +1,57 @@ +""" +This configuration example demonstrates how to +define and use "virtual" plugins. Such plugins +can be defined inside a configuration item. +""" + +from xrlint.node import DatasetNode +from xrlint.rule import RuleContext +from xrlint.rule import RuleOp +from xrlint.rule import define_rule + + +@define_rule("good-title", description="Dataset title should be 'Hello World!'.") +class GoodTitle(RuleOp): + def dataset(self, ctx: RuleContext, node: DatasetNode): + good_title = "Hello World!" + if node.dataset.attrs.get("title") != good_title: + ctx.report( + "Attribute 'title' wrong.", + suggestions=[f"Rename it to {good_title!r}."], + ) + + +# Define more rules here... + + +def export_configs(): + return [ + # Define and use "hello" plugin + { + "plugins": { + "hello": { + "meta": { + "name": "hello", + "version": "1.0.0", + }, + "rules": { + "good-title": GoodTitle, + # Add more rules here... + }, + "configs": { + "recommended": { + "rules": { + "hello/good-title": "warn", + # Configure more rules here... + }, + }, + # Add more configurations here... + }, + }, + } + }, + # Use recommended settings from xrlint + "recommended", + # Use recommended settings from "hello" plugin + "hello/recommended", + ] diff --git a/tests/cli/configs/recommended.yaml b/tests/cli/configs/recommended.yaml index 50e6eec..4ef8545 100644 --- a/tests/cli/configs/recommended.yaml +++ b/tests/cli/configs/recommended.yaml @@ -3,5 +3,5 @@ - recommended - xcube/recommended - rules: - "dataset-title-attr": error - "xcube/single-grid-mapping": off + "dataset-title-attr": "error" + "xcube/single-grid-mapping": "off" diff --git a/tests/cli/test_config.py b/tests/cli/test_config.py index 739bd08..881500b 100644 --- a/tests/cli/test_config.py +++ b/tests/cli/test_config.py @@ -93,7 +93,7 @@ def test_read_config_invalid_arg(self): with pytest.raises( TypeError, match="configuration file must be of type str|Path|PathLike," - " but was None", + " but got None", ): # noinspection PyTypeChecker read_config_list(None) @@ -125,7 +125,7 @@ def test_read_config_yaml_with_type_error(self): match=( "'config.yaml: configuration list must be of" " type ConfigList|list\\[Config|dict|str\\]," - " but was dict'" + " but got dict'" ), ): read_config_list(config_path) @@ -142,7 +142,10 @@ def test_read_config_py_no_export(self): with text_file(self.new_config_py(), py_code) as config_path: with pytest.raises( ConfigError, - match=".py: missing export_configs()", + match=( + "config_1002.py: attribute 'export_configs'" + " not found in module 'config_1002'" + ), ): read_config_list(config_path) @@ -173,7 +176,7 @@ def test_read_config_py_with_invalid_config_list(self): ".py: return value of export_configs\\(\\):" " configuration list must be of type" " ConfigList|list\\[Config|dict|str\\]," - " but was int" + " but got int" ), ): read_config_list(config_path) diff --git a/tests/cli/test_main.py b/tests/cli/test_main.py index 942f12e..06540d2 100644 --- a/tests/cli/test_main.py +++ b/tests/cli/test_main.py @@ -192,7 +192,7 @@ def test_print_config_option(self): "{\n" ' "name": "",\n' ' "plugins": {\n' - ' "__core__": "xrlint.plugins.core"\n' + ' "__core__": "xrlint.plugins.core:export_plugin"\n' " },\n" ' "rules": {\n' ' "dataset-title-attr": 2\n' diff --git a/tests/test_config.py b/tests/test_config.py index 91a7502..046c0bd 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -3,6 +3,7 @@ import pytest from xrlint.config import Config, ConfigList +from xrlint.rule import RuleConfig from xrlint.util.filefilter import FileFilter @@ -42,33 +43,61 @@ def test_from_value_ok(self): } ), ) + self.assertEqual( + Config( + rules={ + "hello/no-spaces-in-titles": RuleConfig(severity=2), + "hello/time-without-tz": RuleConfig(severity=0), + "hello/no-empty-units": RuleConfig( + severity=1, args=(12,), kwargs={"indent": 4} + ), + }, + ), + Config.from_value( + { + "rules": { + "hello/no-spaces-in-titles": 2, + "hello/time-without-tz": "off", + "hello/no-empty-units": ["warn", 12, {"indent": 4}], + }, + } + ), + ) def test_from_value_fails(self): with pytest.raises( - TypeError, match="configuration must be of type dict, but was int" + TypeError, + match=r"config must be of type Config \| dict \| None, but got int", ): Config.from_value(4) + with pytest.raises( - TypeError, match="configuration must be of type dict, but was str" + TypeError, + match=r"config must be of type Config \| dict \| None, but got str", ): Config.from_value("abc") + with pytest.raises( - TypeError, match="configuration must be of type dict, but was tuple" + TypeError, + match=r"config must be of type Config \| dict \| None, but got tuple", ): Config.from_value(()) + with pytest.raises( TypeError, - match="linter_options must be of type dict\\[str,Any\\], but was list", + match=r" config.linter_options must be of type dict.*, but got list", ): Config.from_value({"linter_options": [1, 2, 3]}) + with pytest.raises( - TypeError, match="settings keys must be of type str, but was int" + TypeError, + match=r" keys of config.settings must be of type str, but got int", ): Config.from_value({"settings": {8: 9}}) class ConfigListTest(TestCase): - def test_from_value(self): + def test_from_value_ok(self): config_list = ConfigList.from_value([]) self.assertIsInstance(config_list, ConfigList) self.assertEqual([], config_list.configs) @@ -80,11 +109,13 @@ def test_from_value(self): self.assertIsInstance(config_list, ConfigList) self.assertEqual([Config()], config_list.configs) + # noinspection PyMethodMayBeStatic + def test_from_value_fail(self): with pytest.raises( TypeError, match=( - "configuration list must be of type" - " ConfigList|list\\[Config|dict\\], but was dict" + r"config_list must be of type" + r" ConfigList \| list\[Config \| dict\], but got dict" ), ): ConfigList.from_value({}) diff --git a/tests/test_examples.py b/tests/test_examples.py new file mode 100644 index 0000000..bf1650c --- /dev/null +++ b/tests/test_examples.py @@ -0,0 +1,30 @@ +import unittest +from unittest import TestCase + +from xrlint.config import ConfigList +from xrlint.rule import RuleOp +from xrlint.util.importutil import import_value + + +class ExamplesTest(TestCase): + def test_plugin_config(self): + config_list, _ = import_value( + "examples.plugin_config", "export_configs", factory=ConfigList.from_value + ) + self.assertIsInstance(config_list, ConfigList) + self.assertEqual(3, len(config_list.configs)) + + def test_virtual_plugin_config(self): + config_list, _ = import_value( + "examples.virtual_plugin_config", + "export_configs", + factory=ConfigList.from_value, + ) + self.assertIsInstance(config_list, ConfigList) + self.assertEqual(3, len(config_list.configs)) + + def test_rule_testing(self): + from examples.rule_testing import GoodTitle, GoodTitleTest + + self.assertTrue(issubclass(GoodTitle, RuleOp)) + self.assertTrue(issubclass(GoodTitleTest, unittest.TestCase)) diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 07e8305..b8f2dde 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -6,7 +6,52 @@ from xrlint.plugin import Plugin, PluginMeta from xrlint.processor import ProcessorOp, Processor from xrlint.result import Message -from xrlint.rule import Rule, RuleOp +from xrlint.rule import Rule, RuleOp, define_rule + + +class PluginTest(TestCase): + def test_from_value_ok_plugin(self): + plugin = Plugin(meta=PluginMeta(name="hello")) + self.assertIs(plugin, Plugin.from_value(plugin)) + + def test_from_value_ok_dict(self): + @define_rule() + class MyRule1(RuleOp): + """This is my 1st rule.""" + + @define_rule() + class MyRule2(RuleOp): + """This is my 2nd rule.""" + + plugin = Plugin.from_value( + { + "meta": { + "name": "hello", + "version": "1.2.3", + }, + "rules": { + "r1": MyRule1, + "r2": MyRule2, + }, + "configs": { + "recommended": { + "rules": { + "hello/r1": "warn", + "hello/r2": "error", + }, + }, + }, + } + ) + self.assertIsInstance(plugin, Plugin) + + +class PluginMetaTest(TestCase): + def test_from_value(self): + self.assertEqual( + PluginMeta(name="p", ref="a.b.c:p"), + PluginMeta.from_value({"name": "p", "version": "0.0.0", "ref": "a.b.c:p"}), + ) class PluginDefineRuleDecoratorTest(TestCase): diff --git a/tests/test_processor.py b/tests/test_processor.py new file mode 100644 index 0000000..62ca011 --- /dev/null +++ b/tests/test_processor.py @@ -0,0 +1,103 @@ +from typing import Any +from unittest import TestCase + +import pytest +import xarray as xr + +from xrlint.plugin import Plugin +from xrlint.plugin import PluginMeta +from xrlint.processor import Processor +from xrlint.processor import ProcessorMeta +from xrlint.processor import ProcessorOp +from xrlint.processor import define_processor +from xrlint.result import Message + + +class ProcessorTest(TestCase): + + def test_define_processor(self): + registry = {} + + class MyProcessorOp(ProcessorOp): + def preprocess( + self, file_path: str, opener_options: dict[str, Any] + ) -> list[tuple[xr.Dataset, str]]: + return [] + + def postprocess( + self, messages: list[list[Message]], file_path: str + ) -> list[Message]: + return [] + + processor = define_processor(op_class=MyProcessorOp, registry=registry) + + self.assertTrue(hasattr(MyProcessorOp, "meta")) + # noinspection PyUnresolvedReferences + meta = MyProcessorOp.meta + self.assertIsInstance(meta, ProcessorMeta) + self.assertEqual("my-processor-op", meta.name) + processor2: Processor = registry.get("my-processor-op") + self.assertIs(processor, processor2) + + def test_define_processor_as_decorator(self): + registry = {} + + @define_processor(registry=registry) + class MyProcessorOp(ProcessorOp): + def preprocess( + self, file_path: str, opener_options: dict[str, Any] + ) -> list[tuple[xr.Dataset, str]]: + return [] + + def postprocess( + self, messages: list[list[Message]], file_path: str + ) -> list[Message]: + return [] + + self.assertTrue(hasattr(MyProcessorOp, "meta")) + # noinspection PyUnresolvedReferences + meta = MyProcessorOp.meta + self.assertIsInstance(meta, ProcessorMeta) + self.assertEqual("my-processor-op", meta.name) + processor: Processor = registry.get("my-processor-op") + self.assertIsInstance(processor, Processor) + self.assertIs(MyProcessorOp, processor.op_class) + + # noinspection PyMethodMayBeStatic,PyUnusedLocal + def test_define_processor_as_decorator_fail(self): + + with pytest.raises( + TypeError, + match=( + r"component decorated by define_processor\(\)" + r" must be a subclass of ProcessorOp" + ), + ): + + @define_processor() + class MyProcessorOp: + pass + + def test_define_processor_with_plugin(self): + plugin = Plugin(meta=PluginMeta(name="my-plugin")) + + @plugin.define_processor() + class MyProcessorOp(ProcessorOp): + def preprocess( + self, file_path: str, opener_options: dict[str, Any] + ) -> list[tuple[xr.Dataset, str]]: + return [] + + def postprocess( + self, messages: list[list[Message]], file_path: str + ) -> list[Message]: + return [] + + self.assertTrue(hasattr(MyProcessorOp, "meta")) + # noinspection PyUnresolvedReferences + meta = MyProcessorOp.meta + self.assertIsInstance(meta, ProcessorMeta) + self.assertEqual("my-processor-op", meta.name) + processor: Processor = plugin.processors.get("my-processor-op") + self.assertIsInstance(processor, Processor) + self.assertIs(MyProcessorOp, processor.op_class) diff --git a/tests/test_rule.py b/tests/test_rule.py index 8ab56fc..d0e31c0 100644 --- a/tests/test_rule.py +++ b/tests/test_rule.py @@ -1,8 +1,83 @@ +import unittest from unittest import TestCase import pytest +from xrlint.rule import Rule, define_rule from xrlint.rule import RuleConfig +from xrlint.rule import RuleMeta +from xrlint.rule import RuleOp + + +class MyRule1(RuleOp): + """This is my 1st rule.""" + + +class MyRule2(RuleOp): + """This is my 2nd rule.""" + + +def export_rule(): + return Rule(meta=RuleMeta(name="my-rule-1"), op_class=MyRule1) + + +class RuleTest(TestCase): + def test_from_value_ok_rule(self): + rule = export_rule() + rule2 = Rule.from_value(rule) + self.assertIs(rule, rule2) + + def test_from_value_ok_rule_op(self): + rule = export_rule() + rule2 = Rule.from_value(rule) + self.assertIs(rule, rule2) + + def test_from_value_ok_str(self): + rule = Rule.from_value("tests.test_rule") + self.assertIsInstance(rule, Rule) + self.assertEqual("my-rule-1", rule.meta.name) + self.assertIs(MyRule1, rule.op_class) + + # noinspection PyMethodMayBeStatic + def test_from_value_fails(self): + with pytest.raises( + TypeError, match="value must be of type Rule|str, but got int" + ): + Rule.from_value(73) + + class MyRule3(RuleOp): + """This is my 3rd rule.""" + + with pytest.raises( + ValueError, + match="missing rule metadata, apply define_rule\\(\\) to class MyRule3", + ): + Rule.from_value(MyRule3) + + +class DefineRuleTest(unittest.TestCase): + + def test_decorator(self): + deco = define_rule() + self.assertTrue(callable(deco)) + op_class = deco(MyRule1) + self.assertIs(MyRule1, op_class) + self.assertTrue(hasattr(MyRule1, "meta")) + # noinspection PyUnresolvedReferences + self.assertEqual("my-rule-1", MyRule1.meta.name) + + def test_function(self): + rule = define_rule(op_class=MyRule1) + self.assertIsInstance(rule, Rule) + self.assertEqual("my-rule-1", rule.meta.name) + self.assertIs(MyRule1, rule.op_class) + + def test_with_registry(self): + registry = {} + rule1 = define_rule(op_class=MyRule1, registry=registry) + rule2 = define_rule(op_class=MyRule2, registry=registry) + self.assertIs(rule1, registry["my-rule-1"]) + self.assertIs(rule2, registry["my-rule-2"]) class RuleConfigTest(TestCase): @@ -51,19 +126,19 @@ def test_from_value_ok(self): def test_from_value_fails(self): with pytest.raises( TypeError, - match="rule configuration must be of type int|str|tuple|list, but was None", + match="rule configuration must be of type int|str|tuple|list, but got None", ): RuleConfig.from_value(None) with pytest.raises( ValueError, - match="severity must be one of 'error', 'warn', 'off', 2, 1, 0, but was 4", + match="severity must be one of 'error', 'warn', 'off', 2, 1, 0, but got 4", ): RuleConfig.from_value(4) with pytest.raises( ValueError, match=( "severity must be one of 'error', 'warn', 'off'," - " 2, 1, 0, but was 'debug'" + " 2, 1, 0, but got 'debug'" ), ): RuleConfig.from_value("debug") diff --git a/tests/util/test_codec.py b/tests/util/test_codec.py new file mode 100644 index 0000000..f7ece14 --- /dev/null +++ b/tests/util/test_codec.py @@ -0,0 +1,454 @@ +from dataclasses import dataclass, field +from types import NoneType, UnionType +from typing import ( + Any, + Union, + Optional, + TypeAlias, + get_args, + get_origin, + Mapping, + TYPE_CHECKING, +) +from unittest import TestCase + +import pytest + +from xrlint.util.codec import ( + ValueConstructible, + JsonSerializable, + MappingConstructible, + get_class_parameters, +) + + +@dataclass() +class UselessContainer(ValueConstructible): + pass + + +@dataclass() +class SimpleTypesContainer(MappingConstructible, JsonSerializable): + a: Any = None + b: bool = False + c: int = 0 + d: float = 0.0 + e: str = "abc" + f: type = int + + +@dataclass() +class ComplexTypesContainer(MappingConstructible, JsonSerializable): + p: SimpleTypesContainer = field(default_factory=SimpleTypesContainer) + q: dict[str, bool] = field(default_factory=dict) + r: dict[str, SimpleTypesContainer] = field(default_factory=dict) + s: list[int] = field(default_factory=list) + t: list[SimpleTypesContainer] = field(default_factory=list) + u: int | float | None = None + + +@dataclass() +class UnionTypesContainer(MappingConstructible, JsonSerializable): + m: SimpleTypesContainer | ComplexTypesContainer | None = None + + +@dataclass() +class RequiredPropsContainer(MappingConstructible, JsonSerializable): + x: float + y: float + z: float + + +@dataclass() +class NoTypesContainer(MappingConstructible, JsonSerializable): + u = 0 + v = 0 + w = 0 + + +if TYPE_CHECKING: + # make IDEs and flake8 happy + from xrlint.rule import RuleConfig + from xrlint.plugin import Plugin + + +@dataclass() +class UnresolvedTypesContainer(ComplexTypesContainer, SimpleTypesContainer): + rules: dict[str, "RuleConfig"] = field(default_factory=dict) + plugins: dict[str, "Plugin"] = field(default_factory=dict) + + @classmethod + def _get_forward_refs(cls) -> Optional[Mapping[str, type]]: + from xrlint.rule import RuleConfig + from xrlint.plugin import Plugin + + return { + "RuleConfig": RuleConfig, + "Plugin": Plugin, + } + + +T1: TypeAlias = int | str | Union[bool, None] | None +T2: TypeAlias = Optional[int] +T3: TypeAlias = Optional[Any] + + +class TypingTest(TestCase): + def test_assumptions(self): + # self.assertTrue(isinstance(Any, type)) + self.assertTrue(isinstance(UnionType, type)) + self.assertTrue(not isinstance(Union, type)) + self.assertTrue(not isinstance(Union, UnionType)) + self.assertTrue(Union != UnionType) + + self.assertEqual(None, get_origin("NoTypesContainer")) + self.assertEqual(None, get_origin("dict")) + self.assertEqual(dict, get_origin(dict[str, "NoTypesContainer"])) + self.assertEqual( + (str, "NoTypesContainer"), get_args(dict[str, "NoTypesContainer"]) + ) + + self.assertEqual(Union, get_origin(T1)) + self.assertEqual({bool, int, str, NoneType}, set(get_args(T1))) + + self.assertEqual(Union, get_origin(T2)) + self.assertEqual({int, NoneType}, set(get_args(T2))) + + self.assertEqual(Union, get_origin(T3)) + self.assertEqual({Any, NoneType}, set(get_args(T3))) + + +# noinspection PyMethodMayBeStatic +class JsonSerializableTest(TestCase): + def test_simple_ok(self): + self.assertEqual( + {"a": None, "b": False, "c": 0, "d": 0.0, "e": "abc", "f": "int"}, + SimpleTypesContainer().to_json(), + ) + self.assertEqual( + {"a": "?", "b": True, "c": 12, "d": 34.56, "e": "uvw", "f": "bool"}, + SimpleTypesContainer( + a="?", b=True, c=12, d=34.56, e="uvw", f=bool + ).to_json(), + ) + + def test_complex_ok(self): + container = ComplexTypesContainer( + q=dict(p=True, q=False), + r=dict(u=SimpleTypesContainer(), v=SimpleTypesContainer()), + s=[1, 2, 3], + t=[ + SimpleTypesContainer(c=5, d=6.7), + SimpleTypesContainer(c=8, d=9.1, f=SimpleTypesContainer), + ], + ) + self.assertEqual( + { + "p": {"a": None, "b": False, "c": 0, "d": 0.0, "e": "abc", "f": "int"}, + "q": {"p": True, "q": False}, + "r": { + "u": { + "a": None, + "b": False, + "c": 0, + "d": 0.0, + "e": "abc", + "f": "int", + }, + "v": { + "a": None, + "b": False, + "c": 0, + "d": 0.0, + "e": "abc", + "f": "int", + }, + }, + "s": [1, 2, 3], + "t": [ + {"a": None, "b": False, "c": 5, "d": 6.7, "e": "abc", "f": "int"}, + { + "a": None, + "b": False, + "c": 8, + "d": 9.1, + "e": "abc", + "f": "SimpleTypesContainer", + }, + ], + "u": None, + }, + container.to_json(), + ) + + def test_fail(self): + @dataclass() + class Problematic(JsonSerializable): + data: Any + + with pytest.raises( + TypeError, + match=( + "problematic.data must be of type" + " None|bool|int|float|str|dict|list|tuple, but got object" + ), + ): + Problematic(data=object()).to_json(name="problematic") + + +class ValueConstructibleTest(TestCase): + def test_useless_ok(self): + container = UselessContainer() + self.assertIs(container, UselessContainer.from_value(container)) + + # noinspection PyMethodMayBeStatic + def test_useless_fail(self): + with pytest.raises( + TypeError, + match="uc must be of type UselessContainer, but got None", + ): + UselessContainer.from_value(None, value_name="uc") + + with pytest.raises( + TypeError, + match="uc must be of type UselessContainer, but got bool", + ): + UselessContainer.from_value(True, value_name="uc") + + with pytest.raises( + TypeError, + match="uc must be of type UselessContainer, but got int", + ): + UselessContainer.from_value(1, value_name="uc") + + with pytest.raises( + TypeError, + match="uc must be of type UselessContainer, but got float", + ): + UselessContainer.from_value(0.1, value_name="uc") + + with pytest.raises( + TypeError, + match="uc must be of type UselessContainer, but got str", + ): + UselessContainer.from_value("abc", value_name="uc") + + with pytest.raises( + TypeError, + match="utc must be of type UselessContainer, but got dict", + ): + UselessContainer.from_value({}, value_name="utc") + + with pytest.raises( + TypeError, + match="uc must be of type UselessContainer, but got list", + ): + UselessContainer.from_value([], value_name="uc") + + with pytest.raises( + TypeError, + match="utc must be of type UselessContainer, but got object", + ): + UselessContainer.from_value(object(), value_name="utc") + + with pytest.raises( + TypeError, + match="utc must be of type UselessContainer, but got type", + ): + UselessContainer.from_value(int, value_name="utc") + + with pytest.raises( + TypeError, + match="utc must be of type UselessContainer, but got type", + ): + UselessContainer.from_value(UselessContainer, value_name="utc") + + +class MappingConstructibleTest(TestCase): + + def test_simple_ok(self): + kwargs = dict(a="?", b=True, c=12, d=34.56, e="uvw", f=bytes) + container = SimpleTypesContainer(**kwargs) + self.assertEqual(container, SimpleTypesContainer.from_value(kwargs)) + self.assertIs(container, SimpleTypesContainer.from_value(container)) + + def test_complex_ok(self): + kwargs = { + "p": {"a": None, "b": False, "c": 0, "d": 0.0, "e": "abc"}, + "q": {"p": True, "q": False}, + "r": { + "u": {"a": None, "b": False, "c": 0, "d": 0.0, "e": "abc"}, + "v": {"a": None, "b": False, "c": 0, "d": 0.0, "e": "abc"}, + }, + "s": [1, 2, 3], + "t": [ + {"a": None, "b": False, "c": 5, "d": 6.7, "e": "abc"}, + {"a": None, "b": False, "c": 8, "d": 9.1, "e": "abc", "f": str}, + ], + } + expected_container = ComplexTypesContainer( + p=SimpleTypesContainer(a=None, b=False, c=0, d=0.0, e="abc"), + q={"p": True, "q": False}, + r={ + "u": SimpleTypesContainer(a=None, b=False, c=0, d=0.0, e="abc"), + "v": SimpleTypesContainer(a=None, b=False, c=0, d=0.0, e="abc"), + }, + s=[1, 2, 3], + t=[ + SimpleTypesContainer(a=None, b=False, c=5, d=6.7, e="abc"), + SimpleTypesContainer(a=None, b=False, c=8, d=9.1, e="abc", f=str), + ], + u=None, + ) + self.assertEqual(expected_container, ComplexTypesContainer.from_value(kwargs)) + self.assertIs( + expected_container, ComplexTypesContainer.from_value(expected_container) + ) + + def test_union_ok(self): + expected_union = UnionTypesContainer(m=SimpleTypesContainer()) + self.assertEqual( + expected_union, + UnionTypesContainer.from_value({"m": SimpleTypesContainer()}), + ) + self.assertIs(expected_union, UnionTypesContainer.from_value(expected_union)) + + expected_union = UnionTypesContainer(m=ComplexTypesContainer()) + self.assertEqual( + expected_union, + UnionTypesContainer.from_value({"m": ComplexTypesContainer()}), + ) + self.assertIs(expected_union, UnionTypesContainer.from_value(expected_union)) + + expected_union = UnionTypesContainer(m=None) + self.assertEqual( + expected_union, + UnionTypesContainer.from_value({"m": None}), + ) + self.assertIs(expected_union, UnionTypesContainer.from_value(expected_union)) + + # noinspection PyMethodMayBeStatic + def test_simple_fail(self): + + with pytest.raises( + TypeError, + match=( + r"stc.b must be of type SimpleTypesContainer | dict\[str, Any\]," + r" but got None" + ), + ): + SimpleTypesContainer.from_value({"b": None}, value_name="stc") + + with pytest.raises( + TypeError, match="x is not a member of stc of type SimpleTypesContainer" + ): + SimpleTypesContainer.from_value({"x": 12}, value_name="stc") + + with pytest.raises( + TypeError, match="x, y are not members of stc of type SimpleTypesContainer" + ): + SimpleTypesContainer.from_value({"x": 12, "y": 34}, value_name="stc") + + with pytest.raises( + TypeError, + match=( + "mappings used to instantiate stc of type SimpleTypesContainer" + " must have keys of type str, but found key of type int" + ), + ): + SimpleTypesContainer.from_value({12: "x"}, value_name="stc") + + with pytest.raises( + TypeError, + match=( + r"stc must be of type SimpleTypesContainer | dict\[str, Any\]," + r" but got type" + ), + ): + SimpleTypesContainer.from_value(SimpleTypesContainer, value_name="stc") + + with pytest.raises( + TypeError, + match="stc.f must be of type type, but got str", + ): + SimpleTypesContainer.from_value({"f": "pippo"}, value_name="stc") + + # noinspection PyMethodMayBeStatic + def test_complex_fail(self): + with pytest.raises( + TypeError, + match="keys of ctc.q must be of type str, but got bool", + ): + ComplexTypesContainer.from_value({"q": {True: False}}, value_name="ctc") + + with pytest.raises( + TypeError, + match=r"ctc.q\['x'\] must be of type bool, but got float", + ): + ComplexTypesContainer.from_value({"q": {"x": 2.3}}, value_name="ctc") + + with pytest.raises( + TypeError, + match=r"ctc.s\[1\] must be of type int, but got str", + ): + ComplexTypesContainer.from_value({"s": [1, "x", 3]}, value_name="ctc") + + # noinspection PyMethodMayBeStatic + def test_union_fail(self): + with pytest.raises( + TypeError, + match=( + r"utc must be of type UnionTypesContainer | dict\[str, Any\]," + r" but got int" + ), + ): + UnionTypesContainer.from_value(21, value_name="utc") + + with pytest.raises( + TypeError, + match=( + "utc.m must be of type SimpleTypesContainer" + " | ComplexTypesContainer" + " | None," + " but got str" + ), + ): + UnionTypesContainer.from_value({"m": "pippo"}, value_name="utc") + + def test_get_class_parameters_is_cached(self): + ctc_param = ComplexTypesContainer._get_class_parameters() + stc_param = SimpleTypesContainer._get_class_parameters() + self.assertIs(stc_param, SimpleTypesContainer._get_class_parameters()) + self.assertIs(ctc_param, ComplexTypesContainer._get_class_parameters()) + self.assertIsNot(ctc_param, stc_param) + + +class GetClassParametersTest(TestCase): + + def test_resolves_types(self): + ctc_params = get_class_parameters( + UnresolvedTypesContainer, + forward_refs=UnresolvedTypesContainer._get_forward_refs(), + ) + # order is important! + self.assertEqual( + [ + "a", + "b", + "c", + "d", + "e", + "f", + "p", + "q", + "r", + "s", + "t", + "u", + "rules", + "plugins", + ], + list(ctc_params.keys()), + ) + for k, v in ctc_params.items(): + self.assertIsNotNone(v, msg=f"ctc_params[{k!r}]") diff --git a/tests/util/test_importutil.py b/tests/util/test_importutil.py index 44e69ca..beb3323 100644 --- a/tests/util/test_importutil.py +++ b/tests/util/test_importutil.py @@ -26,7 +26,8 @@ def test_import_submodules(self): ) def test_import_exported_value(self): - core_plugin = import_value( - "xrlint.plugins.core", "export_plugin", Plugin.from_value + plugin, plugin_ref = import_value( + "xrlint.plugins.core", "export_plugin", factory=Plugin.from_value ) - self.assertIsInstance(core_plugin, Plugin) + self.assertIsInstance(plugin, Plugin) + self.assertEqual("xrlint.plugins.core:export_plugin", plugin_ref) diff --git a/tests/util/test_schema.py b/tests/util/test_schema.py index 35935dd..7dda65e 100644 --- a/tests/util/test_schema.py +++ b/tests/util/test_schema.py @@ -25,7 +25,7 @@ def test_type_name_list(self): # noinspection PyTypeChecker,PyMethodMayBeStatic def test_type_name_invalid(self): with pytest.raises( - TypeError, match="type must be of type str|list[str], but was str" + TypeError, match="type must be of type str|list[str], but got str" ): schema(type=str) with pytest.raises( @@ -34,7 +34,7 @@ def test_type_name_invalid(self): "type name must be one of " "'null', 'boolean', 'integer', 'number'," " 'string', 'array', 'object'," - " but was 'float'" + " but got 'float'" ), ): schema(type="float") @@ -42,7 +42,7 @@ def test_type_name_invalid(self): # noinspection PyTypeChecker,PyMethodMayBeStatic def test_type_name_list_invalid(self): with pytest.raises( - TypeError, match="type must be of type str|list[str], but was int" + TypeError, match="type must be of type str|list[str], but got int" ): schema(type=["string", 2]) with pytest.raises( @@ -51,7 +51,7 @@ def test_type_name_list_invalid(self): "type name must be one of" " 'null', 'boolean', 'integer', 'number'," " 'string', 'array', 'object'," - " but was 'list'" + " but got 'list'" ), ): schema(type=["string", "list"]) diff --git a/xrlint/all.py b/xrlint/all.py index 653a1f0..ade9a6a 100644 --- a/xrlint/all.py +++ b/xrlint/all.py @@ -23,12 +23,14 @@ from xrlint.processor import Processor from xrlint.processor import ProcessorMeta from xrlint.processor import ProcessorOp +from xrlint.processor import define_processor from xrlint.rule import Rule from xrlint.rule import RuleConfig from xrlint.rule import RuleContext from xrlint.rule import RuleExit from xrlint.rule import RuleMeta from xrlint.rule import RuleOp +from xrlint.rule import define_rule from xrlint.testing import RuleTest from xrlint.testing import RuleTester from xrlint.version import version @@ -59,12 +61,14 @@ "Processor", "ProcessorMeta", "ProcessorOp", + "define_processor", "Rule", "RuleConfig", "RuleContext", "RuleExit", "RuleMeta", "RuleOp", + "define_rule", "RuleTest", "RuleTester", "version", diff --git a/xrlint/cli/config.py b/xrlint/cli/config.py index aab4eab..95dd6b0 100644 --- a/xrlint/cli/config.py +++ b/xrlint/cli/config.py @@ -85,7 +85,11 @@ def _read_config_python(config_path: str) -> Any: old_sys_path = sys.path sys.path = [module_parent.as_posix()] + sys.path try: - return import_value(module_name, "export_configs", ConfigList.from_value) + return import_value( + module_name, + "export_configs", + factory=ConfigList.from_value, + )[0] except ValueImportError as e: raise ConfigError(config_path, e) from e finally: diff --git a/xrlint/cli/constants.py b/xrlint/cli/constants.py index 96ff6ab..f5d29bd 100644 --- a/xrlint/cli/constants.py +++ b/xrlint/cli/constants.py @@ -1,19 +1,21 @@ -DEFAULT_CONFIG_BASENAME = "xrlint_config" +from typing import Final -DEFAULT_CONFIG_FILE_YAML = f"{DEFAULT_CONFIG_BASENAME}.yaml" -DEFAULT_CONFIG_FILE_JSON = f"{DEFAULT_CONFIG_BASENAME}.json" -DEFAULT_CONFIG_FILE_PY = f"{DEFAULT_CONFIG_BASENAME}.py" +DEFAULT_CONFIG_BASENAME: Final = "xrlint_config" -DEFAULT_CONFIG_FILES = [ +DEFAULT_CONFIG_FILE_YAML: Final = f"{DEFAULT_CONFIG_BASENAME}.yaml" +DEFAULT_CONFIG_FILE_JSON: Final = f"{DEFAULT_CONFIG_BASENAME}.json" +DEFAULT_CONFIG_FILE_PY: Final = f"{DEFAULT_CONFIG_BASENAME}.py" + +DEFAULT_CONFIG_FILES: Final = [ DEFAULT_CONFIG_FILE_YAML, DEFAULT_CONFIG_FILE_JSON, DEFAULT_CONFIG_FILE_PY, ] -DEFAULT_OUTPUT_FORMAT = "simple" -DEFAULT_MAX_WARNINGS = 5 +DEFAULT_OUTPUT_FORMAT: Final = "simple" +DEFAULT_MAX_WARNINGS: Final = 5 -INIT_CONFIG_YAML = "- recommended\n" +INIT_CONFIG_YAML: Final = "- recommended\n" -DEFAULT_GLOBAL_FILES = ["**/*.zarr", "**/*.nc"] -DEFAULT_GLOBAL_IGNORES = [".git", "node_modules"] +DEFAULT_GLOBAL_FILES: Final = ["**/*.zarr", "**/*.nc"] +DEFAULT_GLOBAL_IGNORES: Final = [".git", "node_modules"] diff --git a/xrlint/config.py b/xrlint/config.py index 0c2dc54..a695c9a 100644 --- a/xrlint/config.py +++ b/xrlint/config.py @@ -1,10 +1,10 @@ from dataclasses import dataclass, field from functools import cached_property -from typing import Any, TYPE_CHECKING, Union, Literal +from typing import Any, TYPE_CHECKING, Union, Literal, Sequence from xrlint.constants import CORE_PLUGIN_NAME +from xrlint.util.codec import MappingConstructible, ValueConstructible from xrlint.util.filefilter import FileFilter -from xrlint.util.formatting import format_message_type_of from xrlint.util.todict import ToDictMixin from xrlint.util.merge import ( merge_arrays, @@ -15,6 +15,7 @@ if TYPE_CHECKING: # pragma: no cover + # make IDEs and flake8 happy from xrlint.rule import Rule from xrlint.rule import RuleConfig from xrlint.plugin import Plugin @@ -79,7 +80,7 @@ def merge_configs( @dataclass(frozen=True, kw_only=True) -class Config(ToDictMixin): +class Config(MappingConstructible, ToDictMixin): """Configuration object. A configuration object contains all the information XRLint needs to execute on a set of dataset files. @@ -147,52 +148,8 @@ class Config(ToDictMixin): that should be available to all rules. """ - @classmethod - def from_value(cls, value: Any) -> "Config": - """Convert given `value` into a `Config` object. - - If `value` is already a `Config` then it is returned as-is. - - Args: - value: A `Config` object, a `dict` containing the - configuration properties, or `None` which - converts into an empty configuration. - - Returns: - A `Config` object. - """ - if isinstance(value, Config): - return value - if value is None: - return Config() - if not isinstance(value, dict): - raise TypeError(format_message_type_of("configuration", value, "dict")) - if not value: - return Config() - - files = cls._parse_pattern_list(value, "files") - ignores = cls._parse_pattern_list(value, "ignores") - linter_options = cls._parse_options("linter_options", value) - opener_options = cls._parse_options("opener_options", value) - processor = cls._parse_processor(value) - plugins = cls._parse_plugins(value) - rules = cls._parse_rules(value) - settings = cls._parse_options("settings", value) - - return Config( - name=value.get("name"), - files=files, - ignores=ignores, - linter_options=linter_options, - opener_options=opener_options, - processor=processor, - plugins=plugins, - rules=rules, - settings=settings, - ) - @cached_property - def file_filter(self) -> "FileFilter": + def file_filter(self) -> FileFilter: """The file filter specified by this configuration. May be empty.""" return FileFilter.from_patterns(self.files, self.ignores) @@ -319,76 +276,36 @@ def merge_items(_p1: Plugin, p2: Plugin) -> Plugin: return merge_dicts(plugins1, plugins2, merge_items=merge_items) @classmethod - def _parse_pattern_list(cls, config_dict: dict, name) -> list[str]: - patterns = config_dict.get(name) - if isinstance(patterns, list): - return [cls._parse_pattern(name, v) for v in patterns] - if patterns is not None: - raise TypeError( - format_message_type_of(f"{name} configuration", patterns, "list[str]") - ) + def _from_none(cls, value_name: str) -> "Config": + return Config() @classmethod - def _parse_pattern(cls, name, pattern): - if not isinstance(pattern, str): - raise TypeError( - format_message_type_of(f"pattern in {name} configuration", pattern, str) - ) - return pattern - - @classmethod - def _parse_processor(cls, config_dict: dict) -> Union["ProcessorOp", str, None]: + def _get_forward_refs(cls) -> dict[str, type]: from xrlint.processor import ProcessorOp - - processor = config_dict.get("processor") - if processor is None or isinstance(processor, (str, ProcessorOp)): - return processor - raise TypeError( - format_message_type_of( - "processor configuration", processor, "ProcessorOp|str|None" - ) - ) - - @classmethod - def _parse_plugins(cls, config_dict: dict) -> dict[str, "Plugin"]: from xrlint.plugin import Plugin + from xrlint.rule import Rule + from xrlint.rule import RuleConfig - plugins = config_dict.get("plugins") - if isinstance(plugins, dict): - return {k: Plugin.from_value(v) for k, v in plugins.items()} - if plugins is not None: - raise TypeError( - format_message_type_of("plugins configuration", plugins, "dict") - ) + return { + "ProcessorOp": ProcessorOp, + "Plugin": Plugin, + "Rule": Rule, + "RuleConfig": RuleConfig, + } @classmethod - def _parse_rules(cls, config_dict: dict) -> dict[str, "RuleConfig"]: - from xrlint.rule import RuleConfig - - rules = config_dict.get("rules") - if isinstance(rules, dict): - return {rn: RuleConfig.from_value(rc) for rn, rc in rules.items()} - if rules is not None: - raise TypeError( - format_message_type_of("rules configuration", rules, "dict") - ) + def _get_value_name(cls) -> str: + return "config" @classmethod - def _parse_options(cls, name: str, config_dict: dict) -> dict[str, Any]: - settings = config_dict.get(name) - if isinstance(settings, dict): - for k, v in settings.items(): - if not isinstance(k, str): - raise TypeError(format_message_type_of(f"{name} keys", k, str)) - return {k: v for k, v in settings.items()} - if settings is not None: - raise TypeError(format_message_type_of(name, settings, "dict[str,Any]")) + def _get_value_type_name(cls) -> str: + return "Config | dict | None" def to_dict(self): d = super().to_dict() plugins: dict[str, Plugin] | None = d.get("plugins") if plugins is not None: - d["plugins"] = {k: v.meta.module or "?" for k, v in plugins.items()} + d["plugins"] = {k: v.meta.ref or "?" for k, v in plugins.items()} rules: dict[str, RuleConfig] | None = d.get("rules") if rules is not None: d["rules"] = { @@ -403,69 +320,18 @@ def to_dict(self): @dataclass(frozen=True) -class ConfigList: - """A holder for a list of `Config` objects. +class ConfigList(ValueConstructible): + """A holder for a list of configuration objects of + type [Config][xrlint.config.Config]. You should not use the class constructor directly. Instead, use the `ConfigList.from_value()` function. """ configs: list[Config] = field(default_factory=list) - """The list of `Config` objects.""" - - @classmethod - def from_value(cls, value: Any) -> "ConfigList": - """Convert given `value` into a `ConfigList` object. - - If `value` is already a `ConfigList` then it is returned as-is. - - Args: - value: A `ConfigList` object or `list` of items which can be - converted into `Config` objects including configuration - names of tyype `str`. The latter are resolved against - the plugin configurations seen so far in the list. - - Returns: - A `ConfigList` object. - """ - if isinstance(value, ConfigList): - return value - - if not isinstance(value, list): - raise TypeError( - format_message_type_of( - "configuration list", value, "ConfigList|list[Config|dict|str]" - ) - ) - - configs: list[Config] = [] - plugins: dict[str, Plugin] = {} - for item in value: - if isinstance(item, str): - if CORE_PLUGIN_NAME not in plugins: - plugins.update({CORE_PLUGIN_NAME: get_core_plugin()}) - config = cls._get_named_config(item, plugins) - else: - config = Config.from_value(item) - configs.append(config) - plugins.update(config.plugins if config.plugins else {}) - - return ConfigList(configs) + """The list of configuration objects.""" - @classmethod - def _get_named_config(cls, config_spec: str, plugins: dict[str, "Plugin"]): - plugin_name, config_name = ( - config_spec.split("/", maxsplit=1) - if "/" in config_spec - else (CORE_PLUGIN_NAME, config_spec) - ) - plugin: Plugin | None = plugins.get(plugin_name) - if plugin is None or not plugin.configs or config_name not in plugin.configs: - raise ValueError(f"configuration {config_spec!r} not found") - config_value = plugin.configs[config_name] - return config_value - - def get_global_filter(self, default: FileFilter | None = None) -> "FileFilter": + def get_global_filter(self, default: FileFilter | None = None) -> FileFilter: """Get a global file filter for this configuration list.""" global_filter = FileFilter( default.files if default else (), @@ -506,3 +372,56 @@ def compute_config(self, file_path: str) -> Config | None: rules=config.rules, settings=config.settings, ) + + @classmethod + def from_value(cls, value: Any, value_name: str | None = None) -> "ConfigList": + """Convert given `value` into a `ConfigList` object. + + If `value` is already a `ConfigList` then it is returned as-is. + + Args: + value: A `ConfigList` object or `list` of items which can be + converted into `Config` objects including configuration + names of tyype `str`. The latter are resolved against + the plugin configurations seen so far in the list. + value_name: A value's name. + Returns: + A `ConfigList` object. + """ + return super().from_value(value, value_name=value_name) + + @classmethod + def _from_sequence(cls, value: Sequence, value_name: str) -> "ConfigList": + configs: list[Config] = [] + plugins: dict[str, Plugin] = {} + for item in value: + if isinstance(item, str): + if CORE_PLUGIN_NAME not in plugins: + plugins.update({CORE_PLUGIN_NAME: get_core_plugin()}) + config = cls._get_named_config(item, plugins) + else: + config = Config.from_value(item) + configs.append(config) + plugins.update(config.plugins if config.plugins else {}) + return ConfigList(configs) + + @classmethod + def _get_value_name(cls) -> str: + return "config_list" + + @classmethod + def _get_value_type_name(cls) -> str: + return "ConfigList | list[Config | dict]" + + @classmethod + def _get_named_config(cls, config_spec: str, plugins: dict[str, "Plugin"]): + plugin_name, config_name = ( + config_spec.split("/", maxsplit=1) + if "/" in config_spec + else (CORE_PLUGIN_NAME, config_spec) + ) + plugin: Plugin | None = plugins.get(plugin_name) + if plugin is None or not plugin.configs or config_name not in plugin.configs: + raise ValueError(f"configuration {config_spec!r} not found") + config_value = plugin.configs[config_name] + return config_value diff --git a/xrlint/constants.py b/xrlint/constants.py index 4f4b097..159cbb8 100644 --- a/xrlint/constants.py +++ b/xrlint/constants.py @@ -1,19 +1,22 @@ -SEVERITY_ERROR = 2 -SEVERITY_WARN = 1 -SEVERITY_OFF = 0 +from typing import Final -SEVERITY_NAME_TO_CODE = { +SEVERITY_ERROR: Final = 2 +SEVERITY_WARN: Final = 1 +SEVERITY_OFF: Final = 0 + +SEVERITY_NAME_TO_CODE: Final = { "error": SEVERITY_ERROR, "warn": SEVERITY_WARN, "off": SEVERITY_OFF, } -SEVERITY_CODE_TO_NAME = {v: k for k, v in SEVERITY_NAME_TO_CODE.items()} +SEVERITY_CODE_TO_NAME: Final = {v: k for k, v in SEVERITY_NAME_TO_CODE.items()} +SEVERITY_CODE_TO_CODE: Final = {v: v for v in SEVERITY_NAME_TO_CODE.values()} -SEVERITY_ENUM: dict[int | str, int] = SEVERITY_NAME_TO_CODE | { - v: v for v in SEVERITY_NAME_TO_CODE.values() -} -SEVERITY_ENUM_TEXT = ", ".join(f"{k!r}" for k in SEVERITY_ENUM.keys()) +SEVERITY_ENUM: Final[dict[int | str, int]] = ( + SEVERITY_NAME_TO_CODE | SEVERITY_CODE_TO_CODE +) +SEVERITY_ENUM_TEXT: Final = ", ".join(f"{k!r}" for k in SEVERITY_ENUM.keys()) -MISSING_DATASET_FILE_PATH = "" +MISSING_DATASET_FILE_PATH: Final = "" -CORE_PLUGIN_NAME = "__core__" +CORE_PLUGIN_NAME: Final = "__core__" diff --git a/xrlint/node.py b/xrlint/node.py index 8e4e9b7..7256417 100644 --- a/xrlint/node.py +++ b/xrlint/node.py @@ -70,4 +70,3 @@ class AttrNode(XarrayNode): value: Any """Attribute value.""" - diff --git a/xrlint/plugin.py b/xrlint/plugin.py index 7f8e6aa..2b6eb2d 100644 --- a/xrlint/plugin.py +++ b/xrlint/plugin.py @@ -2,14 +2,14 @@ from typing import Any, Type, Callable, Literal from xrlint.config import Config -from xrlint.processor import Processor, ProcessorOp, register_processor -from xrlint.rule import Rule, RuleOp, register_rule -from xrlint.util.formatting import format_message_type_of +from xrlint.processor import Processor, ProcessorOp, define_processor +from xrlint.rule import Rule, RuleOp, define_rule +from xrlint.util.codec import MappingConstructible from xrlint.util.importutil import import_value -@dataclass(frozen=True, kw_only=True) -class PluginMeta: +@dataclass(kw_only=True) +class PluginMeta(MappingConstructible): """XRLint plugin metadata.""" name: str @@ -18,12 +18,19 @@ class PluginMeta: version: str = "0.0.0" """Plugin version.""" - module: str | None = None - """Plugin module.""" + ref: str | None = None + """Plugin module reference. + Specifies the location from where the plugin can be loaded. + Must have the form ":". + """ + + @classmethod + def _get_value_type_name(cls) -> str: + return "PluginMeta | dict" @dataclass(frozen=True, kw_only=True) -class Plugin: +class Plugin(MappingConstructible): """An XRLint plugin.""" meta: PluginMeta @@ -39,14 +46,6 @@ class Plugin: """A dictionary containing named processors. """ - @classmethod - def from_value(cls, value: Any) -> "Plugin": - if isinstance(value, Plugin): - return value - if isinstance(value, str): - return import_value(value, "export_plugin", Plugin.from_value) - raise TypeError(format_message_type_of("value", value, "Plugin|str")) - def define_rule( self, name: str, @@ -57,8 +56,13 @@ def define_rule( docs_url: str | None = None, op_class: Type[RuleOp] | None = None, ) -> Callable[[Any], Type[RuleOp]] | None: - return register_rule( - self.rules, + """Decorator to define a plugin rule. + The method registers a new rule with the plugin. + + Refer to [define_rule][xrlint.rule.define_rule] + for details. + """ + return define_rule( name=name, version=version, schema=schema, @@ -66,17 +70,36 @@ def define_rule( description=description, docs_url=docs_url, op_class=op_class, + registry=self.rules, ) def define_processor( self, - name: str, + name: str | None = None, version: str = "0.0.0", op_class: Type[ProcessorOp] | None = None, ): - return register_processor( - self.processors, + """Decorator to define a plugin processor. + The method registers a new processor with the plugin. + + Refer to [define_processor][xrlint.processor.define_processor] + for details. + """ + return define_processor( name=name, version=version, op_class=op_class, + registry=self.processors, ) + + @classmethod + def _from_str(cls, value: str, value_name: str) -> "Plugin": + plugin, plugin_ref = import_value( + value, "export_plugin", factory=Plugin.from_value + ) + plugin.meta.ref = plugin_ref + return plugin + + @classmethod + def _get_value_type_name(cls) -> str: + return "Plugin | dict | str" diff --git a/xrlint/plugins/core/rules/__init__.py b/xrlint/plugins/core/rules/__init__.py index def96e7..1282ae3 100644 --- a/xrlint/plugins/core/rules/__init__.py +++ b/xrlint/plugins/core/rules/__init__.py @@ -8,6 +8,6 @@ meta=PluginMeta( name=CORE_PLUGIN_NAME, version=version, - module=__package__.rsplit(".", maxsplit=1)[0], + ref="xrlint.plugins.core:export_plugin", ) ) diff --git a/xrlint/plugins/xcube/rules/__init__.py b/xrlint/plugins/xcube/rules/__init__.py index e7531f2..18fe5ff 100644 --- a/xrlint/plugins/xcube/rules/__init__.py +++ b/xrlint/plugins/xcube/rules/__init__.py @@ -7,6 +7,6 @@ meta=PluginMeta( name="xcube", version=version, - module=__package__.rsplit(".", maxsplit=1)[0], + ref="xrlint.plugins.xcube:export_plugin", ) ) diff --git a/xrlint/plugins/xcube/rules/data_var_colors.py b/xrlint/plugins/xcube/rules/data_var_colors.py index 23a7cbd..5de1c75 100644 --- a/xrlint/plugins/xcube/rules/data_var_colors.py +++ b/xrlint/plugins/xcube/rules/data_var_colors.py @@ -1,5 +1,3 @@ -import numpy as np - from xrlint.node import DataArrayNode from xrlint.plugins.xcube.rules import plugin from xrlint.plugins.xcube.util import is_spatial_var diff --git a/xrlint/processor.py b/xrlint/processor.py index d334248..0727c1b 100644 --- a/xrlint/processor.py +++ b/xrlint/processor.py @@ -1,10 +1,14 @@ from abc import abstractmethod, ABC from dataclasses import dataclass +from inspect import isclass from typing import Type, Any, Callable import xarray as xr from xrlint.result import Message +from xrlint.util.codec import MappingConstructible +from xrlint.util.importutil import import_value +from xrlint.util.naming import to_kebab_case class ProcessorOp(ABC): @@ -45,18 +49,28 @@ def postprocess( @dataclass(frozen=True, kw_only=True) -class ProcessorMeta: +class ProcessorMeta(MappingConstructible): """Processor metadata.""" name: str - """Name of the processor.""" + """Processor name.""" version: str = "0.0.0" - """Version of the processor.""" + """Processor version.""" + + ref: str | None = None + """Processor module reference. + Specifies the location from where the processor can be loaded. + Must have the form ":". + """ + + @classmethod + def _get_value_type_name(cls) -> str: + return "ProcessorMeta | dict" @dataclass(frozen=True, kw_only=True) -class Processor: +class Processor(MappingConstructible): """Processors tell XRLint how to process files other than standard xarray datasets. """ @@ -67,30 +81,97 @@ class Processor: op_class: Type[ProcessorOp] """A class that implements the processor operations.""" - supports_auto_fix: bool = False - """`True` if this processor supports auto-fixing of datasets.""" - + # Not yet: + # supports_auto_fix: bool = False + # """`True` if this processor supports auto-fixing of datasets.""" + + @classmethod + def _from_class( + cls, value: Type[ProcessorOp], name: str | None = None + ) -> "Processor": + # TODO: see code duplication in Rule._from_class() + try: + # Note, the value.meta attribute is set by + # the define_rule + # noinspection PyUnresolvedReferences + return Processor(meta=value.meta, op_class=value) + except AttributeError: + raise ValueError( + f"missing processor metadata, apply define_processor()" + f" to class {value.__name__}" + ) -def register_processor( - registry: dict[str, Processor], - name: str, + @classmethod + def _from_str(cls, value: str, name: str | None = None) -> "Processor": + processor, processor_ref = import_value( + value, + "export_processor", + factory=Processor.from_value, + expected_type=type, + ) + processor.meta.ref = processor_ref + return processor + + @classmethod + def _get_value_type_name(cls) -> str: + return "str | dict | Processor | Type[ProcessorOp]" + + +# TODO: see code duplication in define_rule() +def define_processor( + name: str | None = None, version: str = "0.0.0", + registry: dict[str, Processor] | None = None, op_class: Type[ProcessorOp] | None = None, -) -> Callable[[Any], Type[ProcessorOp]] | None: - def _register_processor(_op_class: Any) -> Type[ProcessorOp]: - from inspect import isclass +) -> Callable[[Any], Type[ProcessorOp]] | Processor: + """Define a processor. + + This function can be used to decorate your processor operation class + definitions. When used as a decorator, the decorated operator class + will receive a `meta` attribute of type + [ProcessorMeta][xrlint.processor.ProcessorMeta]. + In addition, the `registry` if given, will be updated using `name` + as key and a new [Processor][xrlint.processor.Processor] as value. + + Args: + name: Processor name, + see [ProcessorMeta][xrlint.processor.ProcessorMeta]. + version: Processor version, + see [ProcessorMeta][xrlint.processor.ProcessorMeta]. + registry: Processor registry. Can be provided to register the + defined processor using its `name`. + op_class: Processor operation class. Must not be provided + if this function is used as a class decorator. + + Returns: + A decorator function, if `op_class` is `None` otherwise + the value of `op_class`. + + Raises: + TypeError: If either `op_class` or the decorated object is not a + a class derived from [ProcessorOp][xrlint.processor.ProcessorOp]. + """ + def _define_processor( + _op_class: Any, no_deco=False + ) -> Type[ProcessorOp] | Processor: if not isclass(_op_class) or not issubclass(_op_class, ProcessorOp): raise TypeError( f"component decorated by define_processor()" f" must be a subclass of {ProcessorOp.__name__}" ) - meta = ProcessorMeta(name=name, version=version) - registry[name] = Processor(meta=meta, op_class=_op_class) - return _op_class + meta = ProcessorMeta( + name=name or to_kebab_case(_op_class.__name__), + version=version, + ) + setattr(_op_class, "meta", meta) + processor = Processor(meta=meta, op_class=_op_class) + if registry is not None: + registry[meta.name] = processor + return processor if no_deco else _op_class if op_class is None: # decorator case - return _register_processor - - _register_processor(op_class) + return _define_processor + else: + return _define_processor(op_class, no_deco=True) diff --git a/xrlint/result.py b/xrlint/result.py index 62bb107..5de040f 100644 --- a/xrlint/result.py +++ b/xrlint/result.py @@ -34,7 +34,7 @@ class Suggestion(ToDictMixin): """Not used yet.""" @classmethod - def from_value(cls, value: Any) -> "Suggestion": + def from_value(cls, value: Any, name: str | None = None) -> "Suggestion": """Convert given `value` into a `Suggestion` object. If `value` is already a `Suggestion` then it is returned as-is. diff --git a/xrlint/rule.py b/xrlint/rule.py index 4b1f07c..92e5297 100644 --- a/xrlint/rule.py +++ b/xrlint/rule.py @@ -1,14 +1,21 @@ from abc import abstractmethod, ABC +from collections.abc import MutableMapping, Sequence from dataclasses import dataclass, field +from inspect import isclass from typing import Type, Literal, Any, Callable import xarray as xr -from xrlint.constants import SEVERITY_ENUM, SEVERITY_ENUM_TEXT +from xrlint.constants import ( + SEVERITY_ENUM, + SEVERITY_ENUM_TEXT, +) from xrlint.node import DatasetNode, DataArrayNode, AttrsNode, AttrNode from xrlint.result import Suggestion -from xrlint.util.formatting import format_message_type_of, format_message_one_of +from xrlint.util.codec import MappingConstructible, ValueConstructible +from xrlint.util.formatting import format_message_one_of from xrlint.util.importutil import import_value +from xrlint.util.naming import to_kebab_case from xrlint.util.todict import ToDictMixin @@ -117,8 +124,8 @@ def attr(self, context: RuleContext, node: AttrNode) -> None: """ -@dataclass(frozen=True, kw_only=True) -class RuleMeta(ToDictMixin): +@dataclass(kw_only=True) +class RuleMeta(MappingConstructible, ToDictMixin): """Rule metadata.""" name: str @@ -173,9 +180,13 @@ class RuleMeta(ToDictMixin): by the rule’s implementation and its configured severity. """ + @classmethod + def _get_value_type_name(cls) -> str: + return "RuleMeta | dict" + @dataclass(frozen=True) -class Rule: +class Rule(MappingConstructible): """A rule comprises rule metadata and a reference to the class that implements the rule's logic. @@ -195,13 +206,53 @@ class that implements the rule's logic. The class must implement the `RuleOp` interface. """ + @classmethod + def _from_str(cls, value: str, name: str) -> "Rule": + rule, rule_ref = import_value(value, "export_rule", factory=Rule.from_value) + rule.meta.ref = rule_ref + return rule + + @classmethod + def _from_type(cls, value: Type, value_name: str) -> "Rule": + if issubclass(value, RuleOp): + op_class = value + try: + # noinspection PyUnresolvedReferences + # Note, the value.meta attribute is set by + # the define_rule() function. + meta = value.meta + except AttributeError: + raise ValueError( + f"missing rule metadata, apply define_rule()" + f" to class {value.__name__}" + ) + return Rule(meta=meta, op_class=op_class) + super()._from_type(value, value_name) + + @classmethod + def _get_value_type_name(cls) -> str: + return "Rule | dict | str" + @dataclass(frozen=True) -class RuleConfig: +class RuleConfig(ValueConstructible): """A rule configuration. You should not use the class constructor directly. - Instead, use the `RuleConfig.from_value()` function. + Instead, use its [from_value][xrlint.rule.RuleConfig.from_value] + class method. The method's argument value can either be a + rule _severity_, or a list where the first element is a rule + _severity_ and subsequent elements are rule arguments: + + - _severity_ + - `[`_severity_`]` + - `[`_severity_`,` _arg-1 | kwargs_ `]` + - `[`_severity_`,` _arg-1_`,` _arg-2_`,` ...`,` _arg-n | kwargs_`]` + + The rule _severity_ is either + + - one of `"error"`, `"warn"`, `"off"` or + - one of `2` (error), `1` (warn), `0` (off) Args: severity: rule severity, one of `2` (error), `1` (warn), or `0` (off) @@ -219,95 +270,120 @@ class RuleConfig: """Rule operation keyword-arguments.""" @classmethod - def from_value(cls, value: Any) -> "RuleConfig": - """Convert `value` into a `RuleConfig` object. - - A rule configuration value can either be a rule _severity_, - or a list where the first element is a rule - _severity_ and subsequent elements are rule arguments: - - - _severity_ - - `[`_severity_`]` - - `[`_severity_`,` _arg-1 | kwargs_ `]` - - `[`_severity_`,` _arg-1_`,` _arg-2_`,` ...`,` _arg-n | kwargs_`]` - - The rule _severity_ is either - - - one of `"error"`, `"warn"`, `"off"` or - - one of `2` (error), `1` (warn), `0` (off) - - Args: - value: A rule severity or a list where the first element - is a rule severity and subsequent elements are rule - arguments. If the value is already of type `RuleConfig` - it is returned as-is. - - Returns: - A `RuleConfig` object. - """ - if isinstance(value, RuleConfig): - return value - - if isinstance(value, (int, str)): - severity_value, options = value, () - elif isinstance(value, (list, tuple)): - severity_value, options = (value[0], value[1:]) if value else (0, ()) - else: - raise TypeError( - format_message_type_of( - "rule configuration", value, "int|str|tuple|list" - ) - ) - + def _convert_severity(cls, value: int | str) -> Literal[2, 1, 0]: try: - severity = SEVERITY_ENUM[severity_value] + # noinspection PyTypeChecker + return SEVERITY_ENUM[value] except KeyError: raise ValueError( - format_message_one_of("severity", severity_value, SEVERITY_ENUM_TEXT) + format_message_one_of("severity", value, SEVERITY_ENUM_TEXT) ) + @classmethod + def _from_bool(cls, value: bool, name: str) -> "RuleConfig": + return RuleConfig(cls._convert_severity(int(value))) + + @classmethod + def _from_int(cls, value: int, name: str) -> "RuleConfig": + return RuleConfig(cls._convert_severity(value)) + + @classmethod + def _from_str(cls, value: str, name: str) -> "RuleConfig": + return RuleConfig(cls._convert_severity(value)) + + @classmethod + def _from_sequence(cls, value: Sequence, name: str) -> "RuleConfig": + if not value: + raise ValueError() + severity = cls._convert_severity(value[0]) + options = value[1:] if not options: args, kwargs = (), {} elif isinstance(options[-1], dict): args, kwargs = options[:-1], options[-1] else: args, kwargs = options, {} - # noinspection PyTypeChecker return RuleConfig(severity, tuple(args), dict(kwargs)) + @classmethod + def _get_value_name(cls) -> str: + return "rule configuration" + + @classmethod + def _get_value_type_name(cls) -> str: + return "int | str | list" + -def register_rule( - registry: dict[str, Rule], - name: str, +def define_rule( + name: str | None = None, version: str = "0.0.0", schema: dict[str, Any] | list[dict[str, Any]] | bool | None = None, type: Literal["problem", "suggestion", "layout"] | None = None, description: str | None = None, docs_url: str | None = None, + registry: MutableMapping[str, Rule] | None = None, op_class: Type[RuleOp] | None = None, -) -> Callable[[Any], Type[RuleOp]] | None: - def _register_rule(_op_class: Any) -> Type[RuleOp]: - from inspect import isclass +) -> Callable[[Any], Type[RuleOp]] | Rule: + """Define a rule. + This function can be used to decorate your rule operation class + definitions. When used as a decorator, the decorated operator class + will receive a `meta` attribute of type [RuleMeta][xrlint.rule.RuleMeta]. + In addition, the `registry` if given, will be updated using `name` + as key and a new [Rule][xrlint.rule.Rule] as value. + + Args: + name: Rule name, see [RuleMeta][xrlint.rule.RuleMeta]. + version: Rule version, see [RuleMeta][xrlint.rule.RuleMeta]. + schema: Rule operation arguments schema, + see [RuleMeta][xrlint.rule.RuleMeta]. + type: Rule type, see [RuleMeta][xrlint.rule.RuleMeta]. + description: Rule description, + see [RuleMeta][xrlint.rule.RuleMeta]. + docs_url: Rule documentation URL, + see [RuleMeta][xrlint.rule.RuleMeta]. + registry: Rule registry. Can be provided to register the + defined rule using its `name`. + op_class: Rule operation class. Must not be provided + if this function is used as a class decorator. + + Returns: + A decorator function, if `op_class` is `None` otherwise + the value of `op_class`. + + Raises: + TypeError: If either `op_class` or the decorated object is not a + a class derived from [RuleOp][xrlint.rule.RuleOp]. + """ + + def _define_rule(_op_class: Type[RuleOp], no_deco=False) -> Type[RuleOp] | Rule: if not isclass(_op_class) or not issubclass(_op_class, RuleOp): raise TypeError( f"component decorated by define_rule()" f" must be a subclass of {RuleOp.__name__}" ) meta = RuleMeta( - name=name, + name=name or to_kebab_case(_op_class.__name__), version=version, - description=description, + description=description or _op_class.__doc__, docs_url=docs_url, type=type if type is not None else "problem", + # TODO: if schema not given, + # derive it from _op_class' ctor arguments schema=schema, ) - registry[name] = Rule(meta=meta, op_class=_op_class) - return _op_class + # Register rule metadata in rule operation class + setattr(_op_class, "meta", meta) + rule = Rule(meta=meta, op_class=_op_class) + if registry is not None: + # Register rule in rule registry + registry[meta.name] = rule + return rule if no_deco else _op_class if op_class is None: - # decorator case - return _register_rule - - _register_rule(op_class) + # decorator case: return decorated class + return _define_rule + else: + # called as function: return new rule + return _define_rule(op_class, no_deco=True) diff --git a/xrlint/util/codec.py b/xrlint/util/codec.py new file mode 100644 index 0000000..a452a74 --- /dev/null +++ b/xrlint/util/codec.py @@ -0,0 +1,489 @@ +import sys +from collections.abc import Mapping, Sequence +from functools import lru_cache +from inspect import formatannotation, isclass, signature, Parameter +from types import NoneType, UnionType +from typing import ( + Any, + Generic, + TypeVar, + Type, + TypeAlias, + Union, + get_origin, + get_args, + get_type_hints, + Optional, +) + +from xrlint.util.formatting import format_message_type_of + + +JSON_VALUE_TYPE_NAME = "None | bool | int | float | str | dict | list" + +JsonValue: TypeAlias = ( + NoneType | bool | int | float | str | dict[str, "JsonValue"] | list["JsonValue"] +) + +T = TypeVar("T") + +_IS_PYTHON_3_10 = (3, 10) <= sys.version_info < (3, 11) + + +class ValueConstructible(Generic[T]): + """Can be used to make data classes constructible from a value. + + The factory for this purpose is the + class method [from_value][xrlint.util.codec.ValueConstructible.from_value]. + """ + + @classmethod + def from_value(cls, value: Any, value_name: str | None = None) -> T: + """Create an instance of this class from a value. + + Args: + value: The value + value_name: An identifier used for error messages. + Defaults to the value returned by `cls._get_value_name()`. + + Returns: + An instance of this class. + + Raises: + TypeError: If `value` cannot be converted. + """ + value_name = value_name or cls._get_value_name() + if isinstance(value, cls): + return value + if value is None: + return cls._from_none(value_name) + if isinstance(value, bool): + return cls._from_bool(value, value_name) + if isinstance(value, int): + return cls._from_int(value, value_name) + if isinstance(value, float): + return cls._from_float(value, value_name) + if isinstance(value, str): + return cls._from_str(value, value_name) + if isinstance(value, Mapping): + return cls._from_mapping(value, value_name) + if isinstance(value, Sequence): + return cls._from_sequence(value, value_name) + if isinstance(value, type): + if isclass(value) and issubclass(value, cls): + return cls._from_class(value, value_name) + else: + return cls._from_type(value, value_name) + return cls._from_other(value, value_name) + + @classmethod + def _from_none(cls, value_name: str) -> T: + """Create an instance of this class from a `None` value. + The default implementation raises a `TypeError`. + Override to implement a different behaviour. + """ + raise TypeError(cls._format_type_error(None, value_name)) + + @classmethod + def _from_bool(cls, value: bool, value_name: str) -> T: + """Create an instance of this class from a bool value. + The default implementation raises a `TypeError`. + Override to implement a different behaviour. + """ + raise TypeError(cls._format_type_error(value, value_name)) + + @classmethod + def _from_int(cls, value: int, value_name: str) -> T: + """Create an instance of this class from an int value. + The default implementation raises a `TypeError`. + Override to implement a different behaviour. + """ + raise TypeError(cls._format_type_error(value, value_name)) + + @classmethod + def _from_float(cls, value: float, value_name: str) -> T: + """Create an instance of this class from a float value. + The default implementation raises a `TypeError`. + Override to implement a different behaviour. + """ + raise TypeError(cls._format_type_error(value, value_name)) + + @classmethod + def _from_str(cls, value: str, value_name: str) -> T: + """Create an instance of this class from a str value. + The default implementation raises a `TypeError`. + Override to implement a different behaviour. + """ + raise TypeError(cls._format_type_error(value, value_name)) + + @classmethod + def _from_class(cls, value: Type[T], value_name: str) -> T: + """Create an instance of this class from a class value + that is a subclass of `cls`. + The default implementation raises a `TypeError`. + Override to implement a different behaviour. + """ + raise TypeError(cls._format_type_error(value, value_name)) + + @classmethod + def _from_type(cls, value: Type, value_name: str) -> T: + """Create an instance of this class from a type value. + The default implementation raises a `TypeError`. + Override to implement a different behaviour. + """ + raise TypeError(cls._format_type_error(value, value_name)) + + @classmethod + def _from_other(cls, value: Any, value_name: str) -> T: + """Create an instance of this class from a value of + an unknown type. + The default implementation raises a `TypeError`. + Override to implement a different behaviour. + """ + raise TypeError(cls._format_type_error(value, value_name)) + + @classmethod + def _from_mapping(cls, value: Mapping, value_name: str) -> T: + """Create an instance of this class from a mapping value. + The default implementation raises a `TypeError`. + Override to implement a different behaviour. + """ + raise TypeError(cls._format_type_error(value, value_name)) + + @classmethod + def _from_sequence(cls, value: Sequence, value_name: str) -> T: + """Create an instance of this class from a sequence value. + The default implementation raises a `TypeError`. + Override to implement a different behaviour. + """ + raise TypeError(cls._format_type_error(value, value_name)) + + @classmethod + def _convert_value(cls, value: Any, type_annotation: Any, value_name: str) -> Any: + """To be used by subclasses that wish to convert a value with + known type for the target value. + + Args: + value: The value to convert to an instance of the + type specified by `type_annotation`. + type_annotation: The annotation representing the target type. + value_name: An identifier for `value`. + + Returns: + The converted value. + """ + type_origin, type_args = cls._process_annotation(type_annotation) + + if value is None: + # If value is None, ensure value is nullable. + nullable = ( + type_origin is Any + or type_origin is NoneType + or type_origin is Union + and (Any in type_args or NoneType in type_args) + ) + if not nullable: + raise TypeError(cls._format_type_error(value, value_name)) + return None + + if type_origin is Any: + # We cannot do any further type checking, + # therefore return the value as-is + return value + + if type_origin is Union: + # For unions try converting the alternatives. + # Return the first successfully converted value. + assert len(type_args) > 0 + errors = [] + for type_arg in type_args: + try: + return cls._convert_value(value, type_arg, value_name) + except (TypeError, ValueError) as e: + errors.append((type_arg, e)) + # Note, the error message constructed here is suboptimal. + # But we sometimes need all details to trace back to the + # root cause while conversion failed. + raise TypeError( + "all type alternatives failed:\n" + + "\n".join(f" {formatannotation(a)} --> {e}" for a, e in errors) + ) + + # if origin is a real type and value is of type origin + if isclass(type_origin): + # If origin is also a ValueConstructible, we are happy + if issubclass(type_origin, ValueConstructible): + return type_origin.from_value(value, value_name=value_name) + + if isinstance(value, type_origin): + # If value has a compatible type, check first if we + # can take care of special types, i.e., mappings and sequences. + if isinstance(value, (bool, int, float, str)): + # We take a shortcut here. However, str test + # is important, because str is also a sequence! + return value + + if issubclass(type_origin, Mapping): + key_type, item_type = type_args if type_args else (Any, Any) + mapping_value = {} + # noinspection PyUnresolvedReferences + for k, v in value.items(): + if not isinstance(k, key_type): + raise TypeError( + format_message_type_of( + f"keys of {value_name}", k, key_type + ) + ) + mapping_value[k] = cls._convert_value( + v, item_type, f"{value_name}[{k!r}]" + ) + return mapping_value + + if issubclass(type_origin, Sequence): + item_type = type_args[0] if type_args else Any + # noinspection PyTypeChecker + return [ + cls._convert_value(v, item_type, f"{value_name}[{i}]") + for i, v in enumerate(value) + ] + return value + + raise TypeError( + format_message_type_of(value_name, value, formatannotation(type_annotation)) + ) + + @classmethod + @lru_cache(maxsize=1000) + def _get_class_parameters(cls) -> Mapping[str, Parameter]: + """Get the type-resolved parameters of this class' constructor. + The method returns a cached value for `cls`. + + Can be used by subclasses to process annotations. + """ + forward_refs = cls._get_forward_refs() + return get_class_parameters(cls, forward_refs=forward_refs) + + @classmethod + def _get_forward_refs(cls) -> Optional[Mapping[str, type]]: + """Get an extra namespace to be used for resolving parameter type hints. + Called from [ValueConstructible._get_class_parameters][]. + + Can be overridden to provide a namespace for resolving type + forward references for your class. + Defaults to `None`. + """ + return None + + @classmethod + def _get_value_name(cls) -> str: + """Get an identifier for values that can be used to create + instances of this class. + + Can be overridden to provide a custom, user-friendly value name. + Defaults to `"value"`. + """ + return "value" + + @classmethod + def _get_value_type_name(cls) -> str: + """Get a descriptive name for the value types that can + be used to create instances of this class, e.g., `"Rule | str"`. + + Can be overridden to provide a custom, user-friendly type name. + Defaults to this class' name. + """ + return cls.__name__ + + @classmethod + def _process_annotation( + cls, prop_annotation: Any + ) -> tuple[type | UnionType, tuple[type | UnionType, ...]]: + type_origin = get_origin(prop_annotation) + if type_origin is not None: + type_origin = Union if type_origin is UnionType else type_origin + type_args = get_args(prop_annotation) + else: + type_origin = prop_annotation + type_args = () + if _IS_PYTHON_3_10: # pragma: no cover + forward_refs = cls._get_forward_refs() + type_origin = cls._resolve_forward_ref(forward_refs, type_origin) + type_args = tuple( + cls._resolve_forward_ref(forward_refs, type_arg) + for type_arg in type_args + ) + return type_origin, type_args + + @classmethod + def _resolve_forward_ref(cls, namespace, ref: Any) -> Any: # pragma: no cover + if isinstance(ref, str) and namespace: + return namespace.get(ref, ref) + else: + return ref + + @classmethod + def _format_type_error(cls, value: Any, value_name: str) -> str: + return format_message_type_of(value_name, value, cls._get_value_type_name()) + + +class MappingConstructible(Generic[T], ValueConstructible[T]): + """A `ValueConstructible` that accepts both instances of `T` and + mappings (e.g., `dict`) as input values. + """ + + @classmethod + def _from_mapping(cls, mapping: Mapping, value_name: str) -> T: + """Create an instance of this class from a mapping value. + + The default implementation treats the mapping as keyword + arguments passed to the class constructor. + + The use case for this is constructing instances of this class + from JSON-objects. + """ + + mapping_keys = set(mapping.keys()) + properties = cls._get_class_parameters() + + args = [] + kwargs = {} + for prop_name, prop_param in properties.items(): + if prop_name in mapping: + mapping_keys.remove(prop_name) + + if prop_param.annotation is Parameter.empty: + prop_annotation = Any + else: + prop_annotation = prop_param.annotation + + prop_value = cls._convert_value( + mapping[prop_name], + prop_annotation, + value_name=f"{value_name}.{prop_name}", + ) + if prop_param.kind == Parameter.POSITIONAL_ONLY: + args.append(prop_value) + else: + kwargs[prop_name] = prop_value + elif ( + prop_param.default is Parameter.empty + ) or prop_param.kind == Parameter.POSITIONAL_ONLY: + raise TypeError( + f"missing value for required property {value_name}.{prop_name}" + f" of type {cls._get_value_type_name()}" + ) + + if mapping_keys: + invalid_keys = tuple( + filter(lambda k: not isinstance(k, str), mapping.keys()) + ) + if invalid_keys: + invalid_type = type(invalid_keys[0]) + raise TypeError( + f"mappings used to instantiate {value_name}" + f" of type {cls.__name__}" + f" must have keys of type str," + f" but found key of type {invalid_type.__name__}" + ) + + raise TypeError( + f"{', '.join(sorted(mapping_keys))}" + f" {'is not a member' if len(mapping_keys) == 1 else 'are not members'}" + f" of {value_name} of type {cls.__name__}" + ) + + # noinspection PyArgumentList + return cls(*args, **kwargs) + + @classmethod + def _get_value_type_name(cls) -> str: + """Get a descriptive name for the value types that can + be used to create instances of this class, e.g., `"Rule | str"`. + Defaults to `f"{cls.__name__} | dict[str, Any]"`. + """ + return f"{cls.__name__} | dict[str, Any]" + + +def get_class_parameters( + cls, forward_refs: Mapping[str, type] | None = None +) -> Mapping[str, Parameter]: + """Get the type-resolved parameters of this class' constructor. + The returned value is cached. + + Args: + cls: The class to inspect. + forward_refs: Optional extra namespace from which to + resolve forward references. + + Returns: + A mapping from parameter names to parameters. + """ + # Get the signature of the constructor + sig = signature(cls.__init__) + + # Resolve annotations + resolved_hints = get_type_hints(cls.__init__, localns=forward_refs) + + # Process the parameters + resolved_params = {} + for i, (name, param) in enumerate(sig.parameters.items()): + if i > 0: # Skip `self` + annotation = resolved_hints[name] + resolved_params[name] = Parameter( + name, param.kind, default=param.default, annotation=annotation + ) + + return resolved_params + + +class JsonSerializable: + + def to_json(self, name: str | None = None) -> JsonValue: + return self.to_dict(name=name) + + def to_dict(self, name: str | None = None) -> dict[str, JsonValue]: + return self._mapping_to_json(self.__dict__, name or type(self).__name__) + + @classmethod + def _value_to_json(cls, value: Any, name: str) -> JsonValue: + if value is None: + # noinspection PyTypeChecker + return None + if isinstance(value, JsonSerializable): + return value.to_json(name=name) + if isinstance(value, bool): + return bool(value) + if isinstance(value, int): + return int(value) + if isinstance(value, float): + return float(value) + if isinstance(value, str): + return str(value) + if isinstance(value, Mapping): + return cls._mapping_to_json(value, name) + if isinstance(value, Sequence): + return cls._sequence_to_json(value, name) + if isinstance(value, type): + return value.__name__ + raise TypeError(format_message_type_of(name, value, JSON_VALUE_TYPE_NAME)) + + @classmethod + def _mapping_to_json(cls, mapping: Mapping, name: str) -> dict[str, JsonValue]: + return { + k: cls._value_to_json(v, f"{name}.{k}") + for k, v in mapping.items() + if is_public_property_name(k) + } + + @classmethod + def _sequence_to_json(cls, sequence: Sequence, name: str) -> list[JsonValue]: + return [cls._value_to_json(v, f"{name}[{i}]") for i, v in enumerate(sequence)] + + +def is_public_property_name(key: Any) -> bool: + return ( + isinstance(key, str) + and key.isidentifier() + and not key[0].isupper() + and not key[0] == "_" + ) diff --git a/xrlint/util/formatting.py b/xrlint/util/formatting.py index 4265bf7..ff60d8d 100644 --- a/xrlint/util/formatting.py +++ b/xrlint/util/formatting.py @@ -60,13 +60,13 @@ def format_message_one_of(name: str, value: Any, enum_value) -> str: enum_text = enum_value else: enum_text = ", ".join(f"{v!r}" for v in enum_value) - return f"{name} must be one of {enum_text}, but was {value!r}" + return f"{name} must be one of {enum_text}, but got {value!r}" def format_message_type_of(name: str, value: Any, type_value: type | str) -> str: return ( f"{name} must be of type {format_type_of(type_value)}," - f" but was {format_type_of(type(value))}" + f" but got {format_type_of(type(value))}" ) diff --git a/xrlint/util/importutil.py b/xrlint/util/importutil.py index 31429a4..b4dbd27 100644 --- a/xrlint/util/importutil.py +++ b/xrlint/util/importutil.py @@ -1,6 +1,6 @@ import importlib import pathlib -from typing import TypeVar, Callable, Any +from typing import TypeVar, Callable, Any, Type from xrlint.util.formatting import format_message_type_of @@ -41,48 +41,86 @@ def import_submodules(package_name: str, dry_run: bool = False) -> list[str]: def import_value( - module_name: str, - function_name: str, - factory: Callable[[Any], T], + module_ref: str, + attr_ref: str | None = None, + *, + constant: bool = False, + factory: Callable[[Any], T] | None = None, + expected_type: Type[T] | None = None, ) -> T: - """Import an exported value from given module. + """Import an exported value from given module reference. Args: - module_name: Module name. - function_name: Name of the function used to provide - the exported value, e.g., "export_plugin", "export_configs". - factory: - The 1-arg factory function that converts a value - into `T`. + module_ref: Module reference. A string comprising either a fully + qualified module name plus an optional attribute reference + using format ":" or just a module name. + In this case, `attr_ref` should be given. + If it is not given, the module itself will be the exported value. + attr_ref: Attribute reference. Should be given in the case where + `module_ref` does not contain an attribute reference. + Example values are "export_plugin", "export_configs". + constant: If `True` the value is expected to be a constant. + If `False`, the default, the referenced attribute can + be a no-arg callable that yields the actual exported value. + factory: 1-arg factory function that converts a value of unknown + type into `T`. Optional. + expected_type: The expected value type that is a `T`. Optional. Returns: - The imported value of type `T`. + value: The imported value of type `T`. + value_ref: The reference from which the value was imported. Raises: ValueImportError: if the value could not be imported """ - config_module = importlib.import_module(module_name) + if ":" in module_ref: + module_name, attr_ref = module_ref + else: + module_name = module_ref + if attr_ref: + module_ref = f"{module_name}:{attr_ref}" try: - export_function = getattr(config_module, function_name) - except AttributeError: - raise ValueImportError(f"missing {function_name}()") - - if not callable(export_function): + module = importlib.import_module(module_name) + except ImportError as e: + raise ValueImportError( + f"failed to import value from {module_ref!r}: {e}" + ) from e + + attr_value = module + if attr_ref: + attr_names = attr_ref.split(".") + for i, attr_name in enumerate(attr_names): + try: + attr_value = getattr(attr_value, attr_name) + except AttributeError: + raise ValueImportError( + f"attribute {'.'.join(attr_names[:i+1])!r}" + f" not found in module {module_name!r}" + ) + + if not constant and callable(attr_value): + # We don't catch exceptions here, + # because they occur in user land. + # noinspection PyCallingNonCallable + exported_value = attr_value() + else: + exported_value = attr_ref + + if factory is not None: + try: + exported_value = factory(exported_value) + except (ValueError, TypeError) as e: + raise ValueImportError( + f"failed converting value of {module_ref!r}: {e}" + ) from e + + if expected_type is not None and not isinstance(exported_value, expected_type): raise ValueImportError( - format_message_type_of( - function_name, - export_function, - "function", - ) + format_message_type_of(module_ref, exported_value, expected_type) ) - exported_value = export_function() - - try: - return factory(exported_value) - except (ValueError, TypeError) as e: - raise ValueImportError(f"return value of {function_name}(): {e}") from e + return exported_value, module_ref class ValueImportError(ImportError):