From d363c828440d71b44b74c2e8157a92e638406a98 Mon Sep 17 00:00:00 2001 From: Matti Schneider Date: Wed, 17 Mar 2021 16:50:13 +0000 Subject: [PATCH 1/9] Improve invalid parameter type error message --- openfisca_core/parameters.py | 2 +- tests/core/parameter_validation/test_parameter_validation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openfisca_core/parameters.py b/openfisca_core/parameters.py index 0255c1a93d..d97ed1e9a9 100644 --- a/openfisca_core/parameters.py +++ b/openfisca_core/parameters.py @@ -319,7 +319,7 @@ def validate(self, data): ) if not isinstance(value, ALLOWED_PARAM_TYPES): raise ParameterParsingError( - "Invalid value in {} : {}".format(self.name, value), + "Value in {} has type {}, which is not one of the allowed types ({}): {}".format(self.name, type(value), ALLOWED_PARAM_TYPES, value), self.file_path ) diff --git a/tests/core/parameter_validation/test_parameter_validation.py b/tests/core/parameter_validation/test_parameter_validation.py index b4fd57344a..055d12f806 100644 --- a/tests/core/parameter_validation/test_parameter_validation.py +++ b/tests/core/parameter_validation/test_parameter_validation.py @@ -22,7 +22,7 @@ def check(file_name, keywords): @pytest.mark.parametrize("test", [ ('indentation', {'Invalid YAML', 'indentation.yaml', 'line 2', 'mapping values are not allowed'}), ('wrong_scale', {'Unexpected property', 'scale[1]', 'treshold'}), - ('wrong_value', {'Invalid value', 'wrong_value[2015-12-01]', '1A'}), + ('wrong_value', {'not one of the allowed types', 'wrong_value[2015-12-01]', '1A'}), ('unexpected_key_in_parameter', {'Unexpected property', 'unexpected_key'}), ('wrong_type_in_parameter', {'must be of type object'}), ('wrong_type_in_value_history', {'must be of type object'}), From 315e2c6d61b617583e67c713fceeb9cfab12fbc3 Mon Sep 17 00:00:00 2001 From: Matti Schneider Date: Wed, 17 Mar 2021 16:50:23 +0000 Subject: [PATCH 2/9] Allow arrays in parameters --- openfisca_core/parameters.py | 3 ++- tests/core/parameter_validation/array_type.yaml | 6 ++++++ .../parameter_validation/test_parameter_validation.py | 9 +++++++-- 3 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 tests/core/parameter_validation/array_type.yaml diff --git a/openfisca_core/parameters.py b/openfisca_core/parameters.py index d97ed1e9a9..8e89abddaf 100644 --- a/openfisca_core/parameters.py +++ b/openfisca_core/parameters.py @@ -10,6 +10,7 @@ import os import sys import traceback +import typing import yaml import numpy as np @@ -36,7 +37,7 @@ # 'unit' and 'reference' are only listed here for backward compatibility. # It is now recommended to include them in metadata, until a common consensus emerges. COMMON_KEYS = {'description', 'metadata', 'unit', 'reference', 'documentation'} -ALLOWED_PARAM_TYPES = (float, int, bool, type(None)) +ALLOWED_PARAM_TYPES = (float, int, bool, type(None), typing.List) def date_constructor(loader, node): diff --git a/tests/core/parameter_validation/array_type.yaml b/tests/core/parameter_validation/array_type.yaml new file mode 100644 index 0000000000..dfce6552fa --- /dev/null +++ b/tests/core/parameter_validation/array_type.yaml @@ -0,0 +1,6 @@ +description: Array types should be allowed. +values: + 2013-01-01: + value: [ FR ] + 2018-06-01: + value: [ ES, FR, IT, NZ ] diff --git a/tests/core/parameter_validation/test_parameter_validation.py b/tests/core/parameter_validation/test_parameter_validation.py index 055d12f806..5afe149f55 100644 --- a/tests/core/parameter_validation/test_parameter_validation.py +++ b/tests/core/parameter_validation/test_parameter_validation.py @@ -8,7 +8,7 @@ year = 2016 -def check(file_name, keywords): +def check_fails_with_message(file_name, keywords): path = os.path.join(BASE_DIR, file_name) + '.yaml' try: load_parameter_file(path, file_name) @@ -37,7 +37,12 @@ def check(file_name, keywords): ]) def test_parsing_errors(test): with pytest.raises(ParameterParsingError): - check(*test) + check_fails_with_message(*test) + + +def test_array_type(): + path = os.path.join(BASE_DIR, 'array_type.yaml') + load_parameter_file(path, 'array_type') def test_filesystem_hierarchy(): From ec93ac8848448b948d03e196c4a7f00a5e35c76e Mon Sep 17 00:00:00 2001 From: Matti Schneider Date: Tue, 23 Mar 2021 09:03:13 +0000 Subject: [PATCH 3/9] Bump version to 35.2.0 --- CHANGELOG.md | 6 ++++++ setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77ceffb780..5fb8fdea2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +### 35.2.0 [#982](https://github.com/openfisca/openfisca-core/pull/982) + +#### Technical changes + +- Allow parameters to be arrays. + ### 35.1.1 [#981](https://github.com/openfisca/openfisca-core/pull/981) #### Technical changes diff --git a/setup.py b/setup.py index a1ed4b3915..ff4ce2cd90 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ setup( name = 'OpenFisca-Core', - version = '35.1.1', + version = '35.2.0', author = 'OpenFisca Team', author_email = 'contact@openfisca.org', classifiers = [ From 4b36f6f26c3f0681b97ba9d95dee04b30a3779ea Mon Sep 17 00:00:00 2001 From: Matti Schneider Date: Tue, 23 Mar 2021 09:14:28 +0000 Subject: [PATCH 4/9] Unify brackets --- openfisca_web_api/openAPI.yml | 54 +++++++++++++++++------------------ 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/openfisca_web_api/openAPI.yml b/openfisca_web_api/openAPI.yml index 7fe4a64427..1a099a4be0 100644 --- a/openfisca_web_api/openAPI.yml +++ b/openfisca_web_api/openAPI.yml @@ -43,25 +43,25 @@ paths: parameters: - in: "body" name: "Situation" - description: 'Describe the situation (persons and entities). Add the variable you wish to calculate in the proper entity, with null as the value. Learn more in our official documentation: https://openfisca.org/doc/openfisca-web-api/input-output-data.html' + description: "Describe the situation (persons and entities). Add the variable you wish to calculate in the proper entity, with null as the value. Learn more in our official documentation: https://openfisca.org/doc/openfisca-web-api/input-output-data.html" required: true schema: - $ref: '#/definitions/SituationInput' + $ref: "#/definitions/SituationInput" responses: 200: description: "The calculation result is sent back in the response body" headers: - $ref: '#/commons/Headers' + $ref: "#/commons/Headers" schema: - $ref: '#/definitions/SituationOutput' + $ref: "#/definitions/SituationOutput" 404: description: "A variable mentioned in the input situation does not exist in the loaded tax and benefit system. Details are sent back in the response body" headers: - $ref: '#/commons/Headers' + $ref: "#/commons/Headers" 400: description: "The request is invalid. Details about the error are sent back in the response body" headers: - $ref: '#/commons/Headers' + $ref: "#/commons/Headers" /parameters: get: tags: @@ -74,9 +74,9 @@ paths: 200: description: "The list of parameters is sent back in the response body" headers: - $ref: '#/commons/Headers' + $ref: "#/commons/Headers" schema: - $ref: '#/definitions/Parameters' + $ref: "#/definitions/Parameters" /parameter/{parameterID}: get: tags: @@ -95,13 +95,13 @@ paths: 200: description: "The requested parameter's information is sent back in the response body" headers: - $ref: '#/commons/Headers' + $ref: "#/commons/Headers" schema: $ref: "#/definitions/Parameter" 404: description: "The requested parameter does not exist" headers: - $ref: '#/commons/Headers' + $ref: "#/commons/Headers" /variables: get: tags: @@ -114,7 +114,7 @@ paths: 200: description: "The list of variables is sent back in the response body" headers: - $ref: '#/commons/Headers' + $ref: "#/commons/Headers" schema: $ref: "#/definitions/Variables" /variable/{variableID}: @@ -135,13 +135,13 @@ paths: 200: description: "The requested variable's information is sent back in the response body" headers: - $ref: '#/commons/Headers' + $ref: "#/commons/Headers" schema: $ref: "#/definitions/Variable" 404: description: "The requested variable does not exist" headers: - $ref: '#/commons/Headers' + $ref: "#/commons/Headers" /entities: get: tags: @@ -154,7 +154,7 @@ paths: 200: description: "The list of the entities as well as their information is sent back in the response body" headers: - $ref: '#/commons/Headers' + $ref: "#/commons/Headers" schema: $ref: "#/definitions/Entities" /trace: @@ -170,25 +170,25 @@ paths: parameters: - in: "body" name: "Situation" - description: 'Describe the situation (persons and entities). Add the variable you wish to calculate in the proper entity, with null as the value.' + description: "Describe the situation (persons and entities). Add the variable you wish to calculate in the proper entity, with null as the value." required: true schema: - $ref: '#/definitions/SituationInput' + $ref: "#/definitions/SituationInput" responses: 200: description: "The calculation details are sent back in the response body" headers: - $ref: '#/commons/Headers' + $ref: "#/commons/Headers" schema: - $ref: '#/definitions/Trace' + $ref: "#/definitions/Trace" 404: description: "A variable mentioned in the input situation does not exist in the loaded tax and benefit system. Details are sent back in the response body" headers: - $ref: '#/commons/Headers' + $ref: "#/commons/Headers" 400: description: "The request is invalid. Details about the error are sent back in the response body" headers: - $ref: '#/commons/Headers' + $ref: "#/commons/Headers" /spec: get: summary: Provide the API documentation in an OpenAPI format @@ -201,14 +201,14 @@ paths: 200: description: The API documentation is sent back in the response body headers: - $ref: '#/commons/Headers' + $ref: "#/commons/Headers" definitions: Parameter: type: "object" properties: values: - $ref: '#/definitions/Values' + $ref: "#/definitions/Values" brackets: type: "object" additionalProperties: @@ -219,7 +219,7 @@ definitions: type: "object" properties: definition: - type: 'string' + type: "string" metadata: type: "object" description: @@ -237,9 +237,9 @@ definitions: type: "object" properties: description: - type: 'string' + type: "string" href: - type: 'string' + type: "string" Variable: type: "object" @@ -284,9 +284,9 @@ definitions: type: "object" properties: description: - type: 'string' + type: "string" href: - type: 'string' + type: "string" Formula: type: "object" From 8a23aae2af4f2c9d3ef7bdd56e070558c79f27bc Mon Sep 17 00:00:00 2001 From: Matti Schneider Date: Tue, 23 Mar 2021 12:24:19 +0000 Subject: [PATCH 5/9] Fix typo --- openfisca_core/parameters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfisca_core/parameters.py b/openfisca_core/parameters.py index 8e89abddaf..05e41fec0f 100644 --- a/openfisca_core/parameters.py +++ b/openfisca_core/parameters.py @@ -852,7 +852,7 @@ def load_parameter_file(file_path, name = ''): :returns: An instance of :any:`ParameterNode` or :any:`Scale` or :any:`Parameter`. """ if not os.path.exists(file_path): - raise ValueError("{} doest not exist".format(file_path)) + raise ValueError("{} does not exist".format(file_path)) if os.path.isdir(file_path): return ParameterNode(name, directory_path = file_path) data = _load_yaml_file(file_path) From 9d8f6ce71d31fab2167acc2013a3e73b68e313e0 Mon Sep 17 00:00:00 2001 From: Matti Schneider Date: Tue, 23 Mar 2021 13:07:24 +0000 Subject: [PATCH 6/9] Fix indentation --- openfisca_web_api/openAPI.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/openfisca_web_api/openAPI.yml b/openfisca_web_api/openAPI.yml index 1a099a4be0..f6580c5af7 100644 --- a/openfisca_web_api/openAPI.yml +++ b/openfisca_web_api/openAPI.yml @@ -34,12 +34,12 @@ paths: post: summary: "Run a simulation" tags: - - Calculations + - Calculations operationId: "calculate" consumes: - - "application/json" + - "application/json" produces: - - "application/json" + - "application/json" parameters: - in: "body" name: "Situation" @@ -65,11 +65,11 @@ paths: /parameters: get: tags: - - "Parameters" + - "Parameters" summary: "List all available parameters" operationId: "getParameters" produces: - - "application/json" + - "application/json" responses: 200: description: "The list of parameters is sent back in the response body" @@ -161,12 +161,12 @@ paths: post: summary: "Explore a simulation's steps in details." tags: - - Calculations + - Calculations operationId: "trace" consumes: - - "application/json" + - "application/json" produces: - - "application/json" + - "application/json" parameters: - in: "body" name: "Situation" @@ -193,10 +193,10 @@ paths: get: summary: Provide the API documentation in an OpenAPI format tags: - - Documentation + - Documentation operationId: spec produces: - - application/json + - application/json responses: 200: description: The API documentation is sent back in the response body From 4e669618c67a261458a82614b2dd895eab036bb2 Mon Sep 17 00:00:00 2001 From: Matti Schneider Date: Tue, 23 Mar 2021 16:31:22 +0000 Subject: [PATCH 7/9] Document arrays and booleans as parameter values Swagger 2.0 does not support oneOf, see https://github.com/openfisca/openfisca-core/issues/988 --- openfisca_web_api/openAPI.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openfisca_web_api/openAPI.yml b/openfisca_web_api/openAPI.yml index f6580c5af7..d0c52f9a14 100644 --- a/openfisca_web_api/openAPI.yml +++ b/openfisca_web_api/openAPI.yml @@ -303,10 +303,11 @@ definitions: format: "float" Values: + description: All keys are ISO dates. Values can be numbers, booleans, or arrays of a single type (number, boolean or string). type: "object" - additionalProperties: - type: "number" - format: "float" + additionalProperties: true +# propertyNames: # this keyword is part of JSON Schema but is not supported in OpenAPI Specification at the time of writing, see https://swagger.io/docs/specification/data-models/keywords/#unsupported +# pattern: "^[12][0-9]{3}-[01][0-9]-[0-3][0-9]$" # all keys are ISO dates Entities: type: "object" From 4faafff7ee424324098814c4038f173aedfd583f Mon Sep 17 00:00:00 2001 From: Matti Schneider Date: Tue, 23 Mar 2021 17:33:22 +0000 Subject: [PATCH 8/9] Fix title hierarchy Co-authored-by: Mauko Quiroga --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fb8fdea2a..45af5dafab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -### 35.2.0 [#982](https://github.com/openfisca/openfisca-core/pull/982) +## 35.2.0 [#982](https://github.com/openfisca/openfisca-core/pull/982) #### Technical changes @@ -12,7 +12,7 @@ - Fix false negative web API test following an update in the country template used for testing. -### 35.1.0 [#973](https://github.com/openfisca/openfisca-core/pull/973) +## 35.1.0 [#973](https://github.com/openfisca/openfisca-core/pull/973) #### Technical changes From 77d58082526a2201d63a739a9a114c1b7f352180 Mon Sep 17 00:00:00 2001 From: Matti Schneider Date: Tue, 23 Mar 2021 17:34:14 +0000 Subject: [PATCH 9/9] Unify import styles Co-authored-by: Mauko Quiroga --- openfisca_core/parameters.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openfisca_core/parameters.py b/openfisca_core/parameters.py index 05e41fec0f..acb4db26fd 100644 --- a/openfisca_core/parameters.py +++ b/openfisca_core/parameters.py @@ -10,7 +10,6 @@ import os import sys import traceback -import typing import yaml import numpy as np @@ -37,7 +36,7 @@ # 'unit' and 'reference' are only listed here for backward compatibility. # It is now recommended to include them in metadata, until a common consensus emerges. COMMON_KEYS = {'description', 'metadata', 'unit', 'reference', 'documentation'} -ALLOWED_PARAM_TYPES = (float, int, bool, type(None), typing.List) +ALLOWED_PARAM_TYPES = (float, int, bool, type(None), List) def date_constructor(loader, node):