diff --git a/.gitignore b/.gitignore index 00d9f19fb6..f89f5f76c2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,9 +3,11 @@ *.pyc *~ .coverage +.mypy.ini .mypy_cache .mypy_cache* .noseids +.nox .project .pydevproject .pytest_cache diff --git a/README.md b/README.md index a3f18555f1..93597f7e1b 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,19 @@ To run a single test: pytest tests/core/test_parameters.py -k test_parameter_for_period ``` +### Testing with Nox + +In order to test several Python and NumPy versions locally, we recommend you to +install [pipx](https://pypa.github.io/pipx/installation/) and configure it with +your a Python version manager like [pyenv](https://github.com/pyenv/pyenv) and +[nox](https://nox.thea.codes/en/stable/tutorial.html#installation). + +Then run: + +```sh +nox -s +``` + ## Types This repository relies on MyPy for optional dynamic & static type checking. diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 0000000000..7886e41569 --- /dev/null +++ b/noxfile.py @@ -0,0 +1,110 @@ +"""Nox config file.""" + +import nox + +nox.options.reuse_existing_virtualenvs = True +nox.options.stop_on_first_error = True + + +@nox.session(python = ("3.9", "3.8", "3.7"), tags = ("lint", "style")) +@nox.parametrize("numpy", ("1.23", "1.22", "1.21")) +def style(session, numpy): + """Run tests.""" + + if session.python == "3.7" and numpy == "1.22": + return + + session.install("--upgrade", "pip") + session.install(".[dev]") + session.install(f"numpy=={numpy}") + session.run("make", "check-style", external = True) + + +@nox.session(python = ("3.9", "3.8", "3.7"), tags = ("lint", "docs")) +@nox.parametrize("numpy", ("1.23", "1.22", "1.21")) +def docs(session, numpy): + """Run tests.""" + + if session.python == "3.7" and numpy == "1.22": + return + + session.install("--upgrade", "pip") + session.install(".[dev]") + session.install(f"numpy=={numpy}") + session.run("make", "lint-doc", external = True) + + +@nox.session(python = ("3.9", "3.8", "3.7"), tags = ("lint", "mypy")) +@nox.parametrize("numpy", ("1.23", "1.22", "1.21")) +def mypy(session, numpy): + """Run tests.""" + + if session.python == "3.7" and numpy == "1.22": + return + + session.install("--upgrade", "pip") + session.install(".[dev]") + session.install(f"numpy=={numpy}") + session.run("make", "check-types", external = True) + + +@nox.session(python = ("3.9", "3.8", "3.7"), tags = ("lint", "mypy-hxc")) +@nox.parametrize("numpy", ("1.23", "1.22", "1.21")) +def mypy_hxc(session, numpy): + """Run tests.""" + + if session.python == "3.7" and numpy == "1.22": + return + + session.install("--upgrade", "pip") + session.install(".[dev]") + session.install(f"numpy=={numpy}") + session.run("make", "lint-typing-strict", external = True) + + +@nox.session(python = ("3.9", "3.8", "3.7"), tags = ("test", "test-core")) +@nox.parametrize("numpy", ("1.23", "1.22", "1.21")) +def test_core(session, numpy): + """Run tests.""" + + if session.python == "3.7" and numpy == "1.22": + return + + session.install("--upgrade", "pip") + session.install(".[dev]") + session.install(f"numpy=={numpy}") + session.install("--no-deps", "openfisca-country-template") + session.install("--no-deps", "openfisca-extension-template") + session.run("make", "test-core", external = True) + + +@nox.session(python = ("3.9", "3.8", "3.7"), tags = ("test", "test-country")) +@nox.parametrize("numpy", ("1.23", "1.22", "1.21")) +def test_country(session, numpy): + """Run tests.""" + + if session.python == "3.7" and numpy == "1.22": + return + + session.install("--upgrade", "pip") + session.install(".[dev]") + session.install(f"numpy=={numpy}") + session.install("--no-deps", "openfisca-country-template") + session.install("--no-deps", "openfisca-extension-template") + session.run("make", "test-country", external = True) + + +@nox.session(python = ("3.9", "3.8", "3.7"), tags = ("test", "test-extension")) +@nox.parametrize("numpy", ("1.23", "1.22", "1.21")) +def test_extension(session, numpy): + """Run tests.""" + + if session.python == "3.7" and numpy == "1.22": + return + + session.install("--upgrade", "pip") + session.install(".[dev]") + session.install(f"numpy=={numpy}") + session.install("--no-deps", "openfisca-country-template") + session.install("--no-deps", "openfisca-extension-template") + session.run("make", "test-extension", external = True) diff --git a/openfisca_core/commons/__init__.py b/openfisca_core/commons/__init__.py index b3b5d8cbb2..7caa6c397a 100644 --- a/openfisca_core/commons/__init__.py +++ b/openfisca_core/commons/__init__.py @@ -8,6 +8,7 @@ * :func:`.average_rate` * :func:`.concat` * :func:`.empty_clone` + * :func:`.flatten` * :func:`.marginal_rate` * :func:`.stringify_array` * :func:`.switch` @@ -53,11 +54,11 @@ # Official Public API from .formulas import apply_thresholds, concat, switch # noqa: F401 -from .misc import empty_clone, stringify_array # noqa: F401 +from .misc import empty_clone, flatten, stringify_array # noqa: F401 from .rates import average_rate, marginal_rate # noqa: F401 __all__ = ["apply_thresholds", "concat", "switch"] -__all__ = ["empty_clone", "stringify_array", *__all__] +__all__ = ["empty_clone", "flatten", "stringify_array", *__all__] __all__ = ["average_rate", "marginal_rate", *__all__] # Deprecated diff --git a/openfisca_core/commons/formulas.py b/openfisca_core/commons/formulas.py index 4e13b925fb..d1da0fc880 100644 --- a/openfisca_core/commons/formulas.py +++ b/openfisca_core/commons/formulas.py @@ -1,17 +1,13 @@ -from typing import Any, Dict, Sequence, TypeVar +from typing import Any, Dict, Sequence, Union import numpy -from openfisca_core.types import ArrayLike, Array - -T = TypeVar("T") - def apply_thresholds( - input: Array[float], - thresholds: ArrayLike[float], - choices: ArrayLike[float], - ) -> Array[float]: + input: numpy.float_, + thresholds: Sequence[float], + choices: Sequence[float], + ) -> numpy.ndarray: """Makes a choice based on an input and thresholds. From a list of ``choices``, this function selects one of these values @@ -40,7 +36,7 @@ def apply_thresholds( """ - condlist: Sequence[Array[bool]] + condlist: Sequence[Union[bool, numpy.bool_]] condlist = [input <= threshold for threshold in thresholds] if len(condlist) == len(choices) - 1: @@ -57,7 +53,7 @@ def apply_thresholds( return numpy.select(condlist, choices) -def concat(this: ArrayLike[str], that: ArrayLike[str]) -> Array[str]: +def concat(this: Sequence[str], that: Sequence[str]) -> numpy.ndarray: """Concatenates the values of two arrays. Args: @@ -90,9 +86,9 @@ def concat(this: ArrayLike[str], that: ArrayLike[str]) -> Array[str]: def switch( - conditions: Array[Any], - value_by_condition: Dict[float, T], - ) -> Array[T]: + conditions: numpy.ndarray, + value_by_condition: Dict[float, Any], + ) -> numpy.ndarray: """Mimicks a switch statement. Given an array of conditions, returns an array of the same size, @@ -125,4 +121,4 @@ def switch( for condition in value_by_condition.keys() ] - return numpy.select(condlist, value_by_condition.values()) + return numpy.select(condlist, tuple(value_by_condition.values())) diff --git a/openfisca_core/commons/misc.py b/openfisca_core/commons/misc.py index c2edcb25dc..36619aa7e7 100644 --- a/openfisca_core/commons/misc.py +++ b/openfisca_core/commons/misc.py @@ -1,12 +1,14 @@ -from typing import TypeVar +from typing import Any, Iterator, Optional, Sequence, TypeVar -from openfisca_core.types import Array +import itertools + +from openfisca_core import types T = TypeVar("T") def empty_clone(original: T) -> T: - """Creates an empty instance of the same class of the original object. + """Create an empty instance of the same class of the original object. Args: original: An object to clone. @@ -43,8 +45,31 @@ def empty_clone(original: T) -> T: return new -def stringify_array(array: Array) -> str: - """Generates a clean string representation of a numpy array. +def flatten(seqs: Sequence[Sequence[T]]) -> Iterator[T]: + """Flatten a sequence of sequences. + + Args: + seqs: Any sequence of sequences. + + Returns: + An iterator with the values. + + Examples: + >>> list(flatten([(1, 2), (3, 4)])) + [1, 2, 3, 4] + + >>> list(flatten(["ab", "cd"])) + ['a', 'b', 'c', 'd'] + + .. versionadded:: 36.0.0 + + """ + + return itertools.chain.from_iterable(seqs) + + +def stringify_array(array: Optional[types.Array[Any]]) -> str: + """Generate a clean string representation of a numpy array. Args: array: An array. diff --git a/openfisca_core/commons/rates.py b/openfisca_core/commons/rates.py index e9d67c322a..05a94b0396 100644 --- a/openfisca_core/commons/rates.py +++ b/openfisca_core/commons/rates.py @@ -1,15 +1,13 @@ -from typing import Optional +from typing import Optional, Sequence import numpy -from openfisca_core.types import ArrayLike, Array - def average_rate( - target: Array[float], - varying: ArrayLike[float], - trim: Optional[ArrayLike[float]] = None, - ) -> Array[float]: + target: numpy.ndarray, + varying: Sequence[float], + trim: Optional[Sequence[float]] = None, + ) -> numpy.ndarray: """Computes the average rate of a target net income. Given a ``target`` net income, and according to the ``varying`` gross @@ -41,32 +39,31 @@ def average_rate( """ - average_rate: Array[float] - - average_rate = 1 - target / varying + rate: numpy.ndarray + rate = 1 - target / varying if trim is not None: - average_rate = numpy.where( - average_rate <= max(trim), - average_rate, + rate = numpy.where( + rate <= max(trim), + rate, numpy.nan, ) - average_rate = numpy.where( - average_rate >= min(trim), - average_rate, + rate = numpy.where( + rate >= min(trim), + rate, numpy.nan, ) - return average_rate + return rate def marginal_rate( - target: Array[float], - varying: Array[float], - trim: Optional[ArrayLike[float]] = None, - ) -> Array[float]: + target: numpy.ndarray, + varying: numpy.ndarray, + trim: Optional[numpy.ndarray] = None, + ) -> numpy.ndarray: """Computes the marginal rate of a target net income. Given a ``target`` net income, and according to the ``varying`` gross @@ -98,9 +95,9 @@ def marginal_rate( """ - marginal_rate: Array[float] + rate: numpy.ndarray - marginal_rate = ( + rate = ( + 1 - (target[:-1] - target[1:]) / (varying[:-1] - varying[1:]) @@ -108,16 +105,16 @@ def marginal_rate( if trim is not None: - marginal_rate = numpy.where( - marginal_rate <= max(trim), - marginal_rate, + rate = numpy.where( + rate <= max(trim), + rate, numpy.nan, ) - marginal_rate = numpy.where( - marginal_rate >= min(trim), - marginal_rate, + rate = numpy.where( + rate >= min(trim), + rate, numpy.nan, ) - return marginal_rate + return rate diff --git a/openfisca_core/indexed_enums/enum_array.py b/openfisca_core/indexed_enums/enum_array.py index 6a77be57a7..3ffa79b87c 100644 --- a/openfisca_core/indexed_enums/enum_array.py +++ b/openfisca_core/indexed_enums/enum_array.py @@ -63,7 +63,7 @@ def _forbidden_operation(self, other: Any) -> NoReturn: __and__ = _forbidden_operation __or__ = _forbidden_operation - def decode(self) -> numpy.object_: + def decode(self) -> numpy.ndarray: """ Return the array of enum items corresponding to self. @@ -82,7 +82,7 @@ def decode(self) -> numpy.object_: list(self.possible_values), ) - def decode_to_str(self) -> numpy.str_: + def decode_to_str(self) -> numpy.ndarray: """ Return the array of string identifiers corresponding to self. diff --git a/openfisca_core/populations/population.py b/openfisca_core/populations/population.py index cb243aff70..3ec95a4a2f 100644 --- a/openfisca_core/populations/population.py +++ b/openfisca_core/populations/population.py @@ -10,7 +10,7 @@ from openfisca_core import periods, projectors from openfisca_core.holders import Holder, MemoryUsage from openfisca_core.projectors import Projector -from openfisca_core.types import Array, Entity, Period, Role, Simulation +from openfisca_core.types import Entity, Period, Role, Simulation from . import config @@ -21,14 +21,14 @@ class Population: entity: Entity _holders: Dict[str, Holder] count: int - ids: Array[str] + ids: numpy.ndarray def __init__(self, entity: Entity) -> None: self.simulation = None self.entity = entity self._holders = {} self.count = 0 - self.ids = [] + self.ids = numpy.array([]) def clone(self, simulation: Simulation) -> Population: result = Population(self.entity) @@ -38,14 +38,14 @@ def clone(self, simulation: Simulation) -> Population: result.ids = self.ids return result - def empty_array(self) -> Array[float]: + def empty_array(self) -> numpy.ndarray: return numpy.zeros(self.count) def filled_array( self, value: Union[float, bool], dtype: Optional[numpy.dtype] = None, - ) -> Union[Array[float], Array[bool]]: + ) -> numpy.ndarray: return numpy.full(self.count, value, dtype) def __getattr__(self, attribute: str) -> Projector: @@ -64,7 +64,7 @@ def get_index(self, id: str) -> int: def check_array_compatible_with_entity( self, - array: Array[float], + array: numpy.ndarray, ) -> None: if self.count == array.size: return None @@ -95,7 +95,7 @@ def __call__( variable_name: str, period: Optional[Union[int, str, Period]] = None, options: Optional[Sequence[str]] = None, - ) -> Optional[Array[float]]: + ) -> Optional[Sequence[float]]: """ Calculate the variable ``variable_name`` for the entity and the period ``period``, using the variable formula if it exists. @@ -169,7 +169,7 @@ def get_memory_usage( }) @projectors.projectable - def has_role(self, role: Role) -> Optional[Array[bool]]: + def has_role(self, role: Role) -> Optional[Sequence[bool]]: """ Check if a person has a given role within its `GroupEntity` @@ -195,10 +195,10 @@ def has_role(self, role: Role) -> Optional[Array[bool]]: @projectors.projectable def value_from_partner( self, - array: Array[float], + array: numpy.ndarray, entity: Projector, role: Role, - ) -> Optional[Array[float]]: + ) -> Optional[numpy.ndarray]: self.check_array_compatible_with_entity(array) self.entity.check_role_validity(role) @@ -218,9 +218,9 @@ def value_from_partner( def get_rank( self, entity: Population, - criteria: Array[float], + criteria: Sequence[float], condition: bool = True, - ) -> Array[int]: + ) -> numpy.ndarray: """ Get the rank of a person within an entity according to a criteria. The person with rank 0 has the minimum value of criteria. diff --git a/openfisca_core/tracers/computation_log.py b/openfisca_core/tracers/computation_log.py index 9bb830b8d2..285c094f38 100644 --- a/openfisca_core/tracers/computation_log.py +++ b/openfisca_core/tracers/computation_log.py @@ -1,40 +1,31 @@ from __future__ import annotations -import typing -from typing import List, Optional, Union +from typing import Any, Optional, Sequence -import numpy - -from .. import tracers -from openfisca_core.indexed_enums import EnumArray +import sys -if typing.TYPE_CHECKING: - from numpy.typing import ArrayLike +import numpy - Array = Union[EnumArray, ArrayLike] +from openfisca_core import commons, types class ComputationLog: + _full_tracer: types.FullTracer - _full_tracer: tracers.FullTracer - - def __init__(self, full_tracer: tracers.FullTracer) -> None: + def __init__(self, full_tracer: types.FullTracer) -> None: self._full_tracer = full_tracer - def display( - self, - value: Optional[Array], - ) -> str: - if isinstance(value, EnumArray): + def display(self, value: types.Array[Any]) -> str: + if isinstance(value, types.EnumArray): value = value.decode_to_str() - return numpy.array2string(value, max_line_width = float("inf")) + return numpy.array2string(value, max_line_width = sys.maxsize) def lines( self, aggregate: bool = False, max_depth: Optional[int] = None, - ) -> List[str]: + ) -> Sequence[str]: depth = 1 lines_by_tree = [ @@ -43,7 +34,7 @@ def lines( in self._full_tracer.trees ] - return self._flatten(lines_by_tree) + return tuple(commons.flatten(lines_by_tree)) def print_log(self, aggregate = False, max_depth = None) -> None: """ @@ -67,11 +58,14 @@ def print_log(self, aggregate = False, max_depth = None) -> None: def _get_node_log( self, - node: tracers.TraceNode, + node: types.TraceNode, depth: int, aggregate: bool, max_depth: Optional[int], - ) -> List[str]: + ) -> Sequence[str]: + + node_log: Sequence[str] + children_log: Sequence[Sequence[str]] if max_depth is not None and depth > max_depth: return [] @@ -84,12 +78,12 @@ def _get_node_log( in node.children ] - return node_log + self._flatten(children_logs) + return [*node_log, *commons.flatten(children_logs)] def _print_line( self, depth: int, - node: tracers.TraceNode, + node: types.TraceNode, aggregate: bool, max_depth: Optional[int], ) -> str: @@ -114,9 +108,3 @@ def _print_line( formatted_value = self.display(value) return f"{indent}{node.name}<{node.period}> >> {formatted_value}" - - def _flatten( - self, - lists: List[List[str]], - ) -> List[str]: - return [item for list_ in lists for item in list_] diff --git a/openfisca_core/tracers/flat_trace.py b/openfisca_core/tracers/flat_trace.py index d51dd2576b..4d03f4eab1 100644 --- a/openfisca_core/tracers/flat_trace.py +++ b/openfisca_core/tracers/flat_trace.py @@ -1,28 +1,19 @@ from __future__ import annotations -import typing -from typing import Dict, Optional, Union +from typing import Any, Dict import numpy -from openfisca_core import tracers -from openfisca_core.indexed_enums import EnumArray - -if typing.TYPE_CHECKING: - from numpy.typing import ArrayLike - - Array = Union[EnumArray, ArrayLike] - Trace = Dict[str, dict] +from openfisca_core import indexed_enums as enums, types class FlatTrace: + _full_tracer: types.FullTracer - _full_tracer: tracers.FullTracer - - def __init__(self, full_tracer: tracers.FullTracer) -> None: + def __init__(self, full_tracer: types.FullTracer) -> None: self._full_tracer = full_tracer - def key(self, node: tracers.TraceNode) -> str: + def key(self, node: types.TraceNode) -> str: name = node.name period = node.period return f"{name}<{period}>" @@ -52,26 +43,19 @@ def get_serialized_trace(self) -> dict: for key, flat_trace in self.get_trace().items() } - def serialize( - self, - value: Optional[Array], - ) -> Union[Optional[Array], list]: - if isinstance(value, EnumArray): - value = value.decode_to_str() + def serialize(self, value: Any) -> Any: + if not isinstance(value, numpy.ndarray): + return value - if isinstance(value, numpy.ndarray) and \ - numpy.issubdtype(value.dtype, numpy.dtype(bytes)): - value = value.astype(numpy.dtype(str)) + if isinstance(value, enums.EnumArray): + return value.decode_to_str().tolist() - if isinstance(value, numpy.ndarray): - value = value.tolist() + if numpy.issubdtype(value.dtype, numpy.dtype(bytes)): + return value.astype(numpy.dtype(str)).tolist() - return value + return value.tolist() - def _get_flat_trace( - self, - node: tracers.TraceNode, - ) -> Trace: + def _get_flat_trace(self, node: types.TraceNode) -> Dict[str, dict]: key = self.key(node) node_trace = { diff --git a/openfisca_core/tracers/full_tracer.py b/openfisca_core/tracers/full_tracer.py index 6638a789d4..29c90547c6 100644 --- a/openfisca_core/tracers/full_tracer.py +++ b/openfisca_core/tracers/full_tracer.py @@ -1,34 +1,34 @@ from __future__ import annotations import time -import typing -from typing import Dict, Iterator, List, Optional, Union +from typing import Any, Dict, Iterator, List, Optional, Sequence, Union -from .. import tracers +from openfisca_core import types -if typing.TYPE_CHECKING: - from numpy.typing import ArrayLike +from .computation_log import ComputationLog +from .flat_trace import FlatTrace +from .performance_log import PerformanceLog +from .simple_tracer import SimpleTracer +from .trace_node import TraceNode - from openfisca_core.periods import Period - - Stack = List[Dict[str, Union[str, Period]]] +Stack = List[Dict[str, Union[str, types.Period]]] +Value = Union[types.Array[Any], Sequence[Any]] class FullTracer: - - _simple_tracer: tracers.SimpleTracer + _simple_tracer: SimpleTracer _trees: list - _current_node: Optional[tracers.TraceNode] + _current_node: Optional[types.TraceNode] def __init__(self) -> None: - self._simple_tracer = tracers.SimpleTracer() + self._simple_tracer = SimpleTracer() self._trees = [] self._current_node = None def record_calculation_start( self, variable: str, - period: Period, + period: types.Period, ) -> None: self._simple_tracer.record_calculation_start(variable, period) self._enter_calculation(variable, period) @@ -37,9 +37,9 @@ def record_calculation_start( def _enter_calculation( self, variable: str, - period: Period, + period: types.Period, ) -> None: - new_node = tracers.TraceNode( + new_node = TraceNode( name = variable, period = period, parent = self._current_node, @@ -56,13 +56,14 @@ def _enter_calculation( def record_parameter_access( self, parameter: str, - period: Period, - value: ArrayLike, + period: types.Period, + value: Value, ) -> None: if self._current_node is not None: - self._current_node.parameters.append( - tracers.TraceNode(name = parameter, period = period, value = value), + self._current_node.parameters = ( + *self._current_node.parameters, + TraceNode(parameter, period, value = value), ) def _record_start_time( @@ -75,7 +76,7 @@ def _record_start_time( if self._current_node is not None: self._current_node.start = time_in_s - def record_calculation_result(self, value: ArrayLike) -> None: + def record_calculation_result(self, value: Value) -> None: if self._current_node is not None: self._current_node.value = value @@ -103,20 +104,20 @@ def stack(self) -> Stack: return self._simple_tracer.stack @property - def trees(self) -> List[tracers.TraceNode]: + def trees(self) -> List[TraceNode]: return self._trees @property - def computation_log(self) -> tracers.ComputationLog: - return tracers.ComputationLog(self) + def computation_log(self) -> ComputationLog: + return ComputationLog(self) @property - def performance_log(self) -> tracers.PerformanceLog: - return tracers.PerformanceLog(self) + def performance_log(self) -> PerformanceLog: + return PerformanceLog(self) @property - def flat_trace(self) -> tracers.FlatTrace: - return tracers.FlatTrace(self) + def flat_trace(self) -> FlatTrace: + return FlatTrace(self) def _get_time_in_sec(self) -> float: return time.time_ns() / (10**9) @@ -130,7 +131,7 @@ def generate_performance_graph(self, dir_path: str) -> None: def generate_performance_tables(self, dir_path: str) -> None: self.performance_log.generate_performance_tables(dir_path) - def _get_nb_requests(self, tree: tracers.TraceNode, variable: str) -> int: + def _get_nb_requests(self, tree: TraceNode, variable: str) -> int: tree_call = tree.name == variable children_calls = sum( self._get_nb_requests(child, variable) @@ -153,7 +154,7 @@ def get_flat_trace(self) -> dict: def get_serialized_flat_trace(self) -> dict: return self.flat_trace.get_serialized_trace() - def browse_trace(self) -> Iterator[tracers.TraceNode]: + def browse_trace(self) -> Iterator[TraceNode]: def _browse_node(node): yield node diff --git a/openfisca_core/tracers/performance_log.py b/openfisca_core/tracers/performance_log.py index 754d7f8056..f9a9066c47 100644 --- a/openfisca_core/tracers/performance_log.py +++ b/openfisca_core/tracers/performance_log.py @@ -1,23 +1,24 @@ from __future__ import annotations +from typing import Dict, Tuple, Sequence + import csv import importlib.resources import itertools import json import os -import typing -from .. import tracers +from openfisca_core import types -if typing.TYPE_CHECKING: - Trace = typing.Dict[str, dict] - Calculation = typing.Tuple[str, dict] - SortedTrace = typing.List[Calculation] +from .trace_node import TraceNode +Trace = Dict[str, dict] +Calculation = Tuple[str, dict] +SortedTrace = Sequence[Calculation] -class PerformanceLog: - def __init__(self, full_tracer: tracers.FullTracer) -> None: +class PerformanceLog: + def __init__(self, full_tracer: types.FullTracer) -> None: self._full_tracer = full_tracer def generate_graph(self, dir_path: str) -> None: @@ -66,7 +67,7 @@ def generate_performance_tables(self, dir_path: str) -> None: def aggregate_calculation_times( self, flat_trace: Trace, - ) -> typing.Dict[str, dict]: + ) -> Dict[str, dict]: def _aggregate_calculations(calculations: list) -> dict: calculation_count = len(calculations) @@ -85,10 +86,10 @@ def _aggregate_calculations(calculations: list) -> dict: return { 'calculation_count': calculation_count, - 'calculation_time': tracers.TraceNode.round(calculation_time), - 'formula_time': tracers.TraceNode.round(formula_time), - 'avg_calculation_time': tracers.TraceNode.round(calculation_time / calculation_count), - 'avg_formula_time': tracers.TraceNode.round(formula_time / calculation_count), + 'calculation_time': TraceNode.round(calculation_time), + 'formula_time': TraceNode.round(formula_time), + 'avg_calculation_time': TraceNode.round(calculation_time / calculation_count), + 'avg_formula_time': TraceNode.round(formula_time / calculation_count), } def _groupby(calculation: Calculation) -> str: @@ -112,7 +113,7 @@ def _json(self) -> dict: 'children': children, } - def _json_tree(self, tree: tracers.TraceNode) -> dict: + def _json_tree(self, tree: types.TraceNode) -> dict: calculation_total_time = tree.calculation_time() children = [self._json_tree(child) for child in tree.children] @@ -122,7 +123,7 @@ def _json_tree(self, tree: tracers.TraceNode) -> dict: 'children': children, } - def _write_csv(self, path: str, rows: typing.List[dict]) -> None: + def _write_csv(self, path: str, rows: Sequence[dict]) -> None: fieldnames = list(rows[0].keys()) with open(path, 'w') as csv_file: diff --git a/openfisca_core/tracers/trace_node.py b/openfisca_core/tracers/trace_node.py index 93b630886c..9190198645 100644 --- a/openfisca_core/tracers/trace_node.py +++ b/openfisca_core/tracers/trace_node.py @@ -1,30 +1,24 @@ from __future__ import annotations -import dataclasses -import typing - -if typing.TYPE_CHECKING: - import numpy +from typing import Any, Optional, Sequence, Union - from openfisca_core.indexed_enums import EnumArray - from openfisca_core.periods import Period +import dataclasses - Array = typing.Union[EnumArray, numpy.typing.ArrayLike] - Time = typing.Union[float, int] +from openfisca_core import types @dataclasses.dataclass class TraceNode: name: str - period: Period - parent: typing.Optional[TraceNode] = None - children: typing.List[TraceNode] = dataclasses.field(default_factory = list) - parameters: typing.List[TraceNode] = dataclasses.field(default_factory = list) - value: typing.Optional[Array] = None + period: types.Period + parent: Optional[TraceNode] = None + children: Sequence[TraceNode] = dataclasses.field(default_factory = list) + parameters: Sequence[TraceNode] = dataclasses.field(default_factory = list) + value: Optional[Union[types.Array[Any], Sequence[Any]]] = None start: float = 0 end: float = 0 - def calculation_time(self, round_: bool = True) -> Time: + def calculation_time(self, round_: bool = True) -> float: result = self.end - self.start if round_: @@ -47,8 +41,8 @@ def formula_time(self) -> float: return self.round(result) def append_child(self, node: TraceNode) -> None: - self.children.append(node) + self.children = (*self.children, node) @staticmethod - def round(time: Time) -> float: + def round(time: float) -> float: return float(f'{time:.4g}') # Keep only 4 significant figures diff --git a/openfisca_core/types/__init__.py b/openfisca_core/types/__init__.py index 699133aecb..31c7e340b1 100644 --- a/openfisca_core/types/__init__.py +++ b/openfisca_core/types/__init__.py @@ -6,10 +6,10 @@ Official Public API: * :attr:`.Array` - * ``ArrayLike`` - * :attr:`.Cache` * :attr:`.Entity` + * :attr:`.EnumArray` * :attr:`.Formula` + * :attr:`.FullTracer` * :attr:`.Holder` * :attr:`.Instant` * :attr:`.ParameterNodeAtInstant` @@ -19,6 +19,7 @@ * :attr:`.Role`, * :attr:`.Simulation`, * :attr:`.TaxBenefitSystem` + * :attr:`.TraceNode` * :attr:`.Variable` Note: @@ -51,12 +52,13 @@ from ._data import ( # noqa: F401 Array, - ArrayLike, ) from ._domain import ( # noqa: F401 Entity, + EnumArray, Formula, + FullTracer, Holder, Instant, ParameterNodeAtInstant, @@ -66,14 +68,16 @@ Role, Simulation, TaxBenefitSystem, + TraceNode, Variable, ) __all__ = [ "Array", - "ArrayLike", "Entity", + "EnumArray", "Formula", + "FullTracer", "Holder", "Instant", "ParameterNodeAtInstant", @@ -83,5 +87,6 @@ "Role", "Simulation", "TaxBenefitSystem", + "TraceNode", "Variable", ] diff --git a/openfisca_core/types/_data.py b/openfisca_core/types/_data.py index ff7066d43a..572809207f 100644 --- a/openfisca_core/types/_data.py +++ b/openfisca_core/types/_data.py @@ -1,51 +1,3 @@ -from typing import Sequence, TypeVar, Union +from __future__ import annotations -from nptyping import types, NDArray as Array - -import numpy - -T = TypeVar("T", bool, bytes, float, int, object, str) - -types._ndarray_meta._Type = Union[type, numpy.dtype, TypeVar] - -ArrayLike = Union[Array[T], Sequence[T]] -""":obj:`typing.Generic`: Type of any castable to :class:`numpy.ndarray`. - -These include any :obj:`numpy.ndarray` and sequences (like -:obj:`list`, :obj:`tuple`, and so on). - -Examples: - >>> ArrayLike[float] - typing.Union[numpy.ndarray, typing.Sequence[float]] - - >>> ArrayLike[str] - typing.Union[numpy.ndarray, typing.Sequence[str]] - -Note: - It is possible since numpy version 1.21 to specify the type of an - array, thanks to `numpy.typing.NDArray`_:: - - from numpy.typing import NDArray - NDArray[numpy.float64] - - `mypy`_ provides `duck type compatibility`_, so an :obj:`int` is - considered to be valid whenever a :obj:`float` is expected. - -Todo: - * Refactor once numpy version >= 1.21 is used. - -.. versionadded:: 35.5.0 - -.. versionchanged:: 35.6.0 - Moved to :mod:`.types` - -.. _mypy: - https://mypy.readthedocs.io/en/stable/ - -.. _duck type compatibility: - https://mypy.readthedocs.io/en/stable/duck_type_compatibility.html - -.. _numpy.typing.NDArray: - https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray - -""" +from numpy.typing import NDArray as Array # noqa: F401 diff --git a/openfisca_core/types/_domain.py b/openfisca_core/types/_domain.py index 643f27964f..0b61676635 100644 --- a/openfisca_core/types/_domain.py +++ b/openfisca_core/types/_domain.py @@ -10,7 +10,6 @@ class Entity(Protocol): """Entity protocol.""" - key: Any plural: Any @@ -30,6 +29,15 @@ def get_variable( """Abstract method.""" +@typing_extensions.runtime_checkable +class EnumArray(Protocol): + """EnumArray protocol.""" + + @abc.abstractmethod + def decode_to_str(self) -> numpy.ndarray: + """Abstract method.""" + + class Formula(Protocol): """Formula protocol.""" @@ -43,6 +51,12 @@ def __call__( """Abstract method.""" +class FullTracer(Protocol): + """FullTracer protocol.""" + trees: Any + browse_trace: Any + + class Holder(Protocol): """Holder protocol.""" @@ -80,6 +94,7 @@ class Period(Protocol): @abc.abstractmethod def start(self) -> Any: """Abstract method.""" + @property @abc.abstractmethod def unit(self) -> Any: @@ -88,7 +103,6 @@ def unit(self) -> Any: class Population(Protocol): """Population protocol.""" - entity: Any @abc.abstractmethod @@ -98,7 +112,6 @@ def get_holder(self, variable_name: Any) -> Any: class Role(Protocol): """Role protocol.""" - entity: Any subroles: Any @@ -125,7 +138,6 @@ def get_population(self, plural: Optional[Any]) -> Any: class TaxBenefitSystem(Protocol): """TaxBenefitSystem protocol.""" - person_entity: Any @abc.abstractmethod @@ -136,7 +148,17 @@ def get_variable( """Abstract method.""" +class TraceNode(Protocol): + """TraceNode protocol.""" + name: Any + value: Any + period: Any + children: Any + parameters: Any + formula_time: Any + calculation_time: Any + + class Variable(Protocol): """Variable protocol.""" - entity: Any diff --git a/openfisca_tasks/install.mk b/openfisca_tasks/install.mk index d4ffd8667a..50f0a20b55 100644 --- a/openfisca_tasks/install.mk +++ b/openfisca_tasks/install.mk @@ -6,12 +6,12 @@ uninstall: ## Install project's overall dependencies install-deps: @$(call print_help,$@:) - @pip install --upgrade pip twine wheel + @python -m pip install --upgrade pip twine wheel ## Install project's development dependencies. install-edit: @$(call print_help,$@:) - @pip install --upgrade --editable ".[dev]" + @python -m pip install --upgrade --editable ".[dev]" ## Delete builds and compiled python files. clean: diff --git a/openfisca_tasks/lint.mk b/openfisca_tasks/lint.mk index 115c6267bb..5e1a6f3d73 100644 --- a/openfisca_tasks/lint.mk +++ b/openfisca_tasks/lint.mk @@ -11,7 +11,7 @@ check-syntax-errors: . ## Run linters to check for syntax and style errors. check-style: $(shell git ls-files "*.py") @$(call print_help,$@:) - @flake8 $? + @python -m flake8 $? @$(call print_pass,$@:) ## Run linters to check for syntax and style errors in the doc. @@ -29,14 +29,14 @@ lint-doc-%: @## able to integrate documentation improvements progresively. @## @$(call print_help,$(subst $*,%,$@:)) - @flake8 --select=D101,D102,D103,DAR openfisca_core/$* - @pylint openfisca_core/$* + @python -m flake8 --select=D101,D102,D103,DAR openfisca_core/$* + @python -m pylint openfisca_core/$* @$(call print_pass,$@:) ## Run static type checkers for type errors. check-types: @$(call print_help,$@:) - @mypy --package openfisca_core --package openfisca_web_api + @python -m mypy --package openfisca_core --package openfisca_web_api @$(call print_pass,$@:) ## Run static type checkers for type errors (strict). @@ -48,7 +48,7 @@ lint-typing-strict: \ ## Run static type checkers for type errors (strict). lint-typing-strict-%: @$(call print_help,$(subst $*,%,$@:)) - @mypy \ + @python -m mypy \ --cache-dir .mypy_cache-openfisca_core.$* \ --implicit-reexport \ --strict \ @@ -58,5 +58,5 @@ lint-typing-strict-%: ## Run code formatters to correct style errors. format-style: $(shell git ls-files "*.py") @$(call print_help,$@:) - @autopep8 $? + @python -m autopep8 $? @$(call print_pass,$@:) diff --git a/openfisca_tasks/test_code.mk b/openfisca_tasks/test_code.mk index 63fdd4386a..fe83c187f6 100644 --- a/openfisca_tasks/test_code.mk +++ b/openfisca_tasks/test_code.mk @@ -10,8 +10,8 @@ install: install-deps install-edit install-test ## Enable regression testing with template repositories. install-test: @$(call print_help,$@:) - @pip install --upgrade --no-dependencies openfisca-country-template - @pip install --upgrade --no-dependencies openfisca-extension-template + @python -m pip install --upgrade --no-deps openfisca-country-template + @python -m pip install --upgrade --no-deps openfisca-extension-template ## Run openfisca-core & country/extension template tests. test-code: test-core test-country test-extension @@ -31,7 +31,7 @@ test-code: test-core test-country test-extension ## Run openfisca-core tests. test-core: $(shell pytest --quiet --quiet --collect-only 2> /dev/null | cut -f 1 -d ":") @$(call print_help,$@:) - @pytest --quiet --capture=no --xdoctest --xdoctest-verbose=0 \ + @python -m pytest --quiet --capture=no --xdoctest --xdoctest-verbose=0 \ openfisca_core/commons \ openfisca_core/holders \ openfisca_core/types diff --git a/setup.cfg b/setup.cfg index 467e3ede59..651df9dda4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -47,6 +47,7 @@ testpaths = tests ignore_missing_imports = True install_types = True non_interactive = True +plugins = numpy.typing.mypy_plugin [mypy-openfisca_core.commons.tests.*] ignore_errors = True diff --git a/setup.py b/setup.py index 0eb2fa5624..82b53fa731 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ """Package config file. -This file contains all package's metadata, including the current version and -its third-party dependencies. +This file contains all the package's metadata, including the current version +and its third-party dependencies. Note: For integration testing, OpenFisca-Core relies on two other packages, @@ -27,9 +27,8 @@ general_requirements = [ 'dpath >= 1.5.0, < 3.0.0', - 'nptyping == 1.4.4', 'numexpr >= 2.7.0, <= 3.0', - 'numpy >= 1.11, < 1.21', + 'numpy >= 1.21.6, < 1.23.0', 'psutil >= 5.4.7, < 6.0.0', 'pytest >= 4.4.1, < 6.0.0', # For openfisca test 'PyYAML >= 3.10',