Skip to content

Commit 9f49349

Browse files
Backport from #105
1 parent bf4ffb0 commit 9f49349

File tree

9 files changed

+93
-77
lines changed

9 files changed

+93
-77
lines changed

tests/mocked_plugins.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ class MockedEntryPoint:
1717

1818

1919
class MockedPluginA(PluginType):
20-
namespace = "test_namespace"
21-
dynamic = False
20+
namespace = "test_namespace" # pyright: ignore[reportAssignmentType,reportIncompatibleMethodOverride]
21+
dynamic = False # pyright: ignore[reportAssignmentType,reportIncompatibleMethodOverride]
2222

2323
def get_all_configs(
2424
self, known_properties: frozenset[VariantPropertyType] | None

tests/plugins/test_loader.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@
4747

4848

4949
class ClashingPlugin(PluginType):
50-
namespace = "test_namespace"
51-
dynamic = False
50+
namespace = "test_namespace" # pyright: ignore[reportAssignmentType,reportIncompatibleMethodOverride]
51+
dynamic = False # pyright: ignore[reportAssignmentType,reportIncompatibleMethodOverride]
5252

5353
def get_all_configs(
5454
self, known_properties: frozenset[VariantPropertyType] | None
@@ -64,8 +64,8 @@ def get_supported_configs(
6464

6565

6666
class ExceptionPluginBase(PluginType):
67-
namespace = "exception_test"
68-
dynamic = False
67+
namespace = "exception_test" # pyright: ignore[reportAssignmentType,reportIncompatibleMethodOverride]
68+
dynamic = False # pyright: ignore[reportAssignmentType,reportIncompatibleMethodOverride]
6969

7070
returned_value: list[VariantFeatureConfigType]
7171

variantlib/models/provider.py

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ class VariantFeatureConfig(BaseModel):
3636
"validator": lambda val: validate_and(
3737
[
3838
lambda v: validate_type(v, VariantFeatureName),
39-
lambda v: validate_matches_re(v, VALIDATION_FEATURE_NAME_REGEX),
39+
lambda v: validate_matches_re(v, VALIDATION_FEATURE_NAME_REGEX), # pyright: ignore[reportArgumentType]
4040
],
4141
value=val,
4242
)
@@ -49,9 +49,9 @@ class VariantFeatureConfig(BaseModel):
4949
"validator": lambda val: validate_and(
5050
[
5151
lambda v: validate_type(v, list[VariantFeatureValue]),
52-
lambda v: validate_list_matches_re(v, VALIDATION_VALUE_REGEX),
53-
lambda v: validate_list_min_len(v, 1),
54-
lambda v: validate_list_all_unique(v),
52+
lambda v: validate_list_matches_re(v, VALIDATION_VALUE_REGEX), # pyright: ignore[reportArgumentType]
53+
lambda v: validate_list_min_len(v, 1), # pyright: ignore[reportArgumentType]
54+
lambda v: validate_list_all_unique(v), # pyright: ignore[reportArgumentType]
5555
],
5656
value=val,
5757
)
@@ -66,7 +66,7 @@ class ProviderConfig(BaseModel):
6666
"validator": lambda val: validate_and(
6767
[
6868
lambda v: validate_type(v, VariantNamespace),
69-
lambda v: validate_matches_re(v, VALIDATION_NAMESPACE_REGEX),
69+
lambda v: validate_matches_re(v, VALIDATION_NAMESPACE_REGEX), # pyright: ignore[reportArgumentType]
7070
],
7171
value=val,
7272
)
@@ -79,8 +79,8 @@ class ProviderConfig(BaseModel):
7979
"validator": lambda val: validate_and(
8080
[
8181
lambda v: validate_type(v, list[VariantFeatureConfig]),
82-
lambda v: validate_list_min_len(v, 1),
83-
lambda v: validate_list_all_unique(v, keys=["name"]),
82+
lambda v: validate_list_min_len(v, 1), # pyright: ignore[reportArgumentType]
83+
lambda v: validate_list_all_unique(v, keys=["name"]), # pyright: ignore[reportArgumentType]
8484
],
8585
value=val,
8686
),

variantlib/models/variant.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ class VariantProperty(VariantFeature):
102102
"validator": lambda val: validate_and(
103103
[
104104
lambda v: validate_type(v, VariantFeatureValue),
105-
lambda v: validate_matches_re(v, VALIDATION_VALUE_REGEX),
105+
lambda v: validate_matches_re(v, VALIDATION_VALUE_REGEX), # pyright: ignore[reportArgumentType]
106106
],
107107
value=val,
108108
)

variantlib/models/variant_info.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from typing import Any
77

88
from packaging.requirements import Requirement
9+
910
from variantlib.constants import VALIDATION_FEATURE_NAME_REGEX
1011
from variantlib.constants import VALIDATION_NAMESPACE_REGEX
1112
from variantlib.constants import VALIDATION_PROVIDER_ENABLE_IF_REGEX

variantlib/plugins/_subprocess.py

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from functools import reduce
99
from itertools import groupby
1010
from typing import TYPE_CHECKING
11+
from typing import Any
1112

1213
# The following imports are replaced with temporary paths by the plugin
1314
# loader. We are using the original imports here to facilitate static
@@ -43,6 +44,7 @@ def load_plugins(plugin_apis: list[str]) -> Generator[PluginType]:
4344

4445
# plugin-api can either be a callable (e.g. a class to instantiate
4546
# or a function to call) or a ready object
47+
plugin_instance: PluginType
4648
if callable(plugin_callable):
4749
try:
4850
# Instantiate the plugin
@@ -52,12 +54,12 @@ def load_plugins(plugin_apis: list[str]) -> Generator[PluginType]:
5254
f"Instantiating the plugin from {plugin_api!r} failed: {exc}"
5355
) from exc
5456
else:
55-
plugin_instance = plugin_callable
57+
plugin_instance = plugin_callable # pyright: ignore[reportAssignmentType]
5658

5759
# We cannot use isinstance() here since some of the PluginType methods
5860
# are optional. Instead, we use @abstractmethod decorator to naturally
5961
# annotate required methods, and the remaining methods are optional.
60-
required_attributes = PluginType.__abstractmethods__
62+
required_attributes = PluginType.__abstractmethods__ # pyright: ignore[reportAttributeAccessIssue]
6163
if missing_attributes := required_attributes.difference(dir(plugin_instance)):
6264
raise TypeError(
6365
f"{plugin_api!r} does not meet the PluginType prototype: "
@@ -96,7 +98,6 @@ def group_properties_by_plugin(
9698
def main() -> int:
9799
parser = argparse.ArgumentParser()
98100
parser.add_argument(
99-
"-p",
100101
"--plugin-api",
101102
action="append",
102103
help="Load specified plugin API",
@@ -107,7 +108,7 @@ def main() -> int:
107108
plugins = dict(zip(args.plugin_api, load_plugins(args.plugin_api)))
108109
namespace_map = {plugin.namespace: plugin for plugin in plugins.values()}
109110

110-
retval = {}
111+
retval: dict[str, Any] = {}
111112
for command, command_args in commands.items():
112113
if command == "namespaces":
113114
assert not command_args

variantlib/plugins/loader.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
from packaging.markers import Marker
1919
from packaging.markers import default_environment
20+
2021
from variantlib.constants import VALIDATION_PROVIDER_PLUGIN_API_REGEX
2122
from variantlib.errors import NoPluginFoundError
2223
from variantlib.errors import PluginError
@@ -85,6 +86,7 @@ def _call_subprocess(
8586
self, plugin_apis: list[str], commands: dict[str, Any]
8687
) -> dict[str, dict[str, Any]]:
8788
with TemporaryDirectory(prefix="variantlib") as temp_dir:
89+
# Copy `variantlib/plugins/loader.py` into the temp_dir
8890
script = Path(temp_dir) / "loader.py"
8991
script.write_bytes(
9092
(importlib.resources.files(__package__) / "_subprocess.py")
@@ -95,9 +97,13 @@ def _call_subprocess(
9597
b"from _variantlib_validators_base",
9698
)
9799
)
100+
101+
# Copy `variantlib/protocols.py` into the temp_dir
98102
(Path(temp_dir) / "_variantlib_protocols.py").write_bytes(
99103
(importlib.resources.files("variantlib") / "protocols.py").read_bytes()
100104
)
105+
106+
# Copy `variantlib/validators/base.py` into the temp_dir
101107
(Path(temp_dir) / "_variantlib_validators_base.py").write_bytes(
102108
(
103109
importlib.resources.files("variantlib.validators") / "base.py"
@@ -106,7 +112,7 @@ def _call_subprocess(
106112

107113
args = []
108114
for plugin_api in plugin_apis:
109-
args += ["-p", plugin_api]
115+
args += ["--plugin-api", plugin_api]
110116

111117
process = subprocess.run( # noqa: S603
112118
[self._python_executable, script, *args],

variantlib/resolver/filtering.py

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
from collections.abc import Generator
1515

1616
from variantlib.protocols import VariantFeatureValue
17+
from variantlib.protocols import VariantNamespace
18+
1719

1820
logger = logging.getLogger(__name__)
1921

@@ -188,51 +190,57 @@ def filter_variants_by_property(
188190

189191
# We group allowed properties by their namespace and feature:
190192
# => only one match per group is required.
191-
allowed_props_dict: dict[VariantFeature, set[VariantFeatureValue]] = defaultdict(
192-
set
193-
)
193+
# Note: This step is required for the OR match within one VariantFeature
194+
allowed_props_dict: dict[
195+
tuple[VariantNamespace, VariantFeatureValue], set[VariantFeatureValue]
196+
] = defaultdict(set)
194197
for vprop in allowed_properties:
195-
allowed_props_dict[VariantFeature(vprop.namespace, vprop.feature)].add(
196-
vprop.value
197-
)
198+
allowed_props_dict[(vprop.namespace, vprop.feature)].add(vprop.value)
198199

199200
def _should_include(vdesc: VariantDescription) -> bool:
200201
"""
201202
Check if any of the namespaces in the variant description are not allowed.
202203
"""
203204
validate_type(vdesc, VariantDescription)
204205

205-
vprops_dict: dict[VariantFeature, set[VariantFeatureValue]] = defaultdict(set)
206+
vdesc_prop_dict: dict[
207+
tuple[VariantNamespace, VariantFeatureValue], set[VariantFeatureValue]
208+
] = defaultdict(set)
206209
for vprop in vdesc.properties:
207-
vprops_dict[VariantFeature(vprop.namespace, vprop.feature)].add(vprop.value)
210+
vdesc_prop_dict[(vprop.namespace, vprop.feature)].add(vprop.value)
208211

209-
for vfeat_tuple, property_values in vprops_dict.items():
210-
if not (allowed_props := allowed_props_dict.get(vfeat_tuple)):
212+
for (ns, vfeat_name), property_values in vdesc_prop_dict.items():
213+
if not (allowed_props := allowed_props_dict.get((ns, vfeat_name))):
211214
# If there are no allowed properties for this feature, we reject
212215
# the variant.
213216
logger.info(
214217
"Variant `%(vhash)s` has been rejected because the feature "
215-
"`%(feature)s` is not supported by any of the allowed properties.",
218+
"`%(ns)s :: %(feature)s` has no allowed properties.",
216219
{
217220
"vhash": vdesc.hexdigest,
218-
"feature": vfeat_tuple.feature,
221+
"ns": ns,
222+
"feature": vfeat_name,
219223
},
220224
)
221225
return False
222226

223227
for property_value in property_values:
224228
if property_value in allowed_props:
225229
break
230+
226231
else:
227232
# We never broke out of the loop, meaning no allowed property
228233
# matched. Consequently, we reject this variant.
229234
logger.info(
230-
"Variant `%(vhash)s` has been rejected because the feature "
231-
"`%(feature)s` is not supported by any of the allowed "
232-
"properties.",
235+
"Variant `%(vhash)s` has been rejected because the none of the "
236+
"variant properties are compatible with this platform:"
237+
"\n\t- %(vprops)s",
233238
{
234239
"vhash": vdesc.hexdigest,
235-
"feature": vfeat_tuple.feature,
240+
"vprops": "\n\t- ".join(
241+
f"`{ns} :: {vfeat_name} :: {val}`"
242+
for val in property_values
243+
),
236244
},
237245
)
238246
return False

variantlib/resolver/sorting.py

Lines changed: 42 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
from __future__ import annotations
22

3+
import contextlib
34
import logging
45
import sys
6+
from collections import defaultdict
57
from itertools import chain
68
from itertools import groupby
79

@@ -196,14 +198,19 @@ def sort_variants_descriptions(
196198
validate_type(vdescs, list[VariantDescription])
197199
validate_type(property_priorities, list[VariantProperty])
198200

199-
vprops: set[VariantProperty] = set()
200-
for vdesc in vdescs:
201-
vprops.update(vdesc.properties)
202-
203201
# Pre-compute the property hash for the property priorities
204202
# This is used to speed up the sorting process.
205203
# The property_hash is used to compare the `VariantProperty` objects
206-
property_hash_priorities = [vprop.property_hash for vprop in property_priorities]
204+
# property_hash_priorities = [vprop.property_hash for vprop in property_priorities]
205+
property_lookup_table: dict[
206+
tuple[VariantNamespace, VariantFeatureName], list[VariantFeatureValue]
207+
] = defaultdict(list)
208+
209+
for vprop in property_priorities:
210+
property_lookup_table[(vprop.namespace, vprop.feature)].append(vprop.value)
211+
212+
property_lookup_table = dict(property_lookup_table)
213+
lookup_table_size = len(property_lookup_table)
207214

208215
def _get_rank_tuple(vdesc: VariantDescription) -> tuple[int, ...]:
209216
"""
@@ -213,42 +220,35 @@ def _get_rank_tuple(vdesc: VariantDescription) -> tuple[int, ...]:
213220
:return: Rank tuple[int, ...] of the `VariantDescription` object.
214221
"""
215222

216-
if vdesc.is_null_variant():
217-
# return the tuple that represents the lowest priority
218-
return tuple(sys.maxsize for _ in property_hash_priorities)
219-
220-
# --------------------------- Implementation Notes --------------------------- #
221-
# - `property_hash_priorities` is ordered. It's a list.
222-
# - `vdesc_prop_hashes` is unordered. It's a set.
223-
#
224-
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Performance Notes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ #
225-
# * Only `property_hash_priorities` needs to be ordered. The set is used for
226-
# performance reasons.
227-
# * `vdesc.properties` hashes are pre-computed and saved to avoid recomputing
228-
# them multiple times.
229-
# * `property_priorities` are also pre-hashed to avoid recomputing them
230-
# ---------------------------------------------------------------------------- #
231-
232-
# using a set for performance reason: O(1) access time.
233-
vdesc_prop_hashes = {vprop.property_hash for vprop in vdesc.properties}
234-
235-
# N-dimensional tuple with tuple[N] of 1 or sys.maxsize
236-
# 1 if the property is present in the `VariantDescription` object,
237-
# sys.maxsize if not present.
238-
# This is used to sort the `VariantDescription` objects based on their
239-
# `VariantProperty`s.
240-
ranking_tuple = tuple(
241-
1 if vprop_hash in vdesc_prop_hashes else sys.maxsize
242-
for vprop_hash in property_hash_priorities
243-
)
244-
245-
if sum(1 for x in ranking_tuple if x != sys.maxsize) != len(vdesc.properties):
246-
raise ValidationError(
247-
f"VariantDescription {vdesc} contains properties not in the property "
248-
"priorities list - this should not happen. Filtering should be applied "
249-
"first."
250-
)
251-
252-
return ranking_tuple
223+
# Initialization of the tuple at the maximum on every dimension: lowest priority
224+
ranking_array = [sys.maxsize for _ in range(lookup_table_size)]
225+
226+
if not vdesc.is_null_variant():
227+
vdesc_feature_indexes: set[int] = set()
228+
for vprop in vdesc.properties:
229+
vprop_key = (vprop.namespace, vprop.feature)
230+
231+
try:
232+
# The following can not raises `ValueError` otherwise the vdesc
233+
# would have been filtered out.
234+
vprop_idx = list(property_lookup_table.keys()).index(vprop_key)
235+
vdesc_feature_indexes.add(vprop_idx)
236+
except ValueError as e:
237+
raise ValidationError("Filtering should be applied first.") from e
238+
239+
with contextlib.suppress(ValueError):
240+
# This call will raise `ValueError` if `vprop.value` is not in the
241+
# list of allowed properties.
242+
ranking_array[vprop_idx] = min(
243+
ranking_array[vprop_idx],
244+
property_lookup_table[vprop_key].index(vprop.value),
245+
)
246+
247+
# We check that the variant has found a compatible property for each
248+
# Variant Feature, otherwise it should have been filtered out.
249+
if any(ranking_array[idx] == sys.maxsize for idx in vdesc_feature_indexes):
250+
raise ValidationError("Filtering should be applied first.")
251+
252+
return tuple(ranking_array)
253253

254254
return sorted(vdescs, key=_get_rank_tuple)

0 commit comments

Comments
 (0)