Skip to content

Commit 2d38429

Browse files
pcrespovpcrespov
authored
🐛 Fixes e2e failures: super and _getattr_ and adds mypy to the dev-toolkit (#167)
* super and _getattr_ * fixes self * adds mypy * fixes with mypy * minor mypy
1 parent 688deb1 commit 2d38429

File tree

10 files changed

+182
-39
lines changed

10 files changed

+182
-39
lines changed

‎clients/python/Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,9 @@ install-doc: _check_venv_active install-dev ## install packages for generating d
122122
pylint: _check_venv_active ## runs linter (only to check errors. SEE .pylintrc enabled)
123123
pylint --rcfile "$(PYTHON_DIR)/.pylintrc" -v "$(ARTIFACTS_DIR)/client"
124124

125+
.PHONY: mypy
126+
mypy: $(SCRIPTS_DIR)/mypy.bash $(PYTHON_DIR)/mypy.ini ## runs mypy python static type-checker on this services's code. Use AFTER make install-*
127+
@$(SCRIPTS_DIR)/mypy.bash client
125128

126129
.PHONY: test-dev
127130
test-dev: _check_venv_active ## runs tests during development
@@ -179,6 +182,7 @@ e2e-shell: guard-GH_TOKEN ## shell for running e2e tests
179182
-c "python -m venv /tmp/venv && source /tmp/venv/bin/activate && cd clients/python \
180183
&& make install-test && cd test/e2e && python ci/install_osparc_python_client.py && exec /bin/bash"
181184

185+
182186
## DOCKER -------------------------------------------------------------------------------
183187

184188
.env: .env-template ## creates .env file from defaults in .env-devel

‎clients/python/client/osparc/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,4 +114,4 @@
114114
"UsersApi",
115115
"UsersGroup",
116116
"ValidationError",
117-
) # type: ignore
117+
)

‎clients/python/client/osparc/_files_api.py‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@ def __init__(self, api_client: Optional[ApiClient] = None):
4343
api_client (ApiClient, optinal): osparc.ApiClient object
4444
"""
4545
super().__init__(api_client)
46-
self._super = super(FilesApi, self)
4746
user: Optional[str] = self.api_client.configuration.username
4847
passwd: Optional[str] = self.api_client.configuration.password
4948
self._auth: Optional[httpx.BasicAuth] = (
@@ -53,13 +52,13 @@ def __init__(self, api_client: Optional[ApiClient] = None):
5352
)
5453

5554
def download_file(
56-
self, file_id: str, *, destination_folder: Optional[Path] = None
55+
self, file_id: str, *, destination_folder: Optional[Path] = None, **kwargs
5756
) -> str:
5857
if destination_folder is not None and not destination_folder.is_dir():
5958
raise RuntimeError(
6059
f"destination_folder: {destination_folder} must be a directory"
6160
)
62-
downloaded_file: Path = Path(super().download_file(file_id))
61+
downloaded_file: Path = Path(super().download_file(file_id, **kwargs))
6362
if destination_folder is not None:
6463
dest_file: Path = destination_folder / downloaded_file.name
6564
while dest_file.is_file():
@@ -74,14 +73,20 @@ def download_file(
7473
return str(downloaded_file.resolve())
7574

7675
def upload_file(
77-
self, file: Union[str, Path], timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS
76+
self,
77+
file: Union[str, Path],
78+
timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS,
79+
**kwargs,
7880
):
7981
return asyncio.run(
80-
self.upload_file_async(file=file, timeout_seconds=timeout_seconds)
82+
self.upload_file_async(file=file, timeout_seconds=timeout_seconds, **kwargs)
8183
)
8284

8385
async def upload_file_async(
84-
self, file: Union[str, Path], timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS
86+
self,
87+
file: Union[str, Path],
88+
timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS,
89+
**kwargs,
8590
) -> File:
8691
if isinstance(file, str):
8792
file = Path(file)
@@ -100,9 +105,9 @@ async def upload_file_async(
100105
filesize=file.stat().st_size,
101106
sha256_checksum=checksum,
102107
)
103-
client_upload_schema: ClientFileUploadData = self._super.get_upload_links(
104-
client_file=client_file, _request_timeout=timeout_seconds
105-
) # type: ignore
108+
client_upload_schema: ClientFileUploadData = super().get_upload_links(
109+
client_file=client_file, _request_timeout=timeout_seconds, **kwargs
110+
)
106111
chunk_size: int = client_upload_schema.upload_schema.chunk_size
107112
links: FileUploadData = client_upload_schema.upload_schema.links
108113
url_iter: Iterator[Tuple[int, str]] = enumerate(
@@ -124,7 +129,7 @@ async def upload_file_async(
124129
file_chunk_generator(file, chunk_size),
125130
total=n_urls,
126131
disable=(not _logger.isEnabledFor(logging.INFO)),
127-
): # type: ignore
132+
):
128133
index, url = next(url_iter)
129134
uploaded_parts.append(
130135
await self._upload_chunck(

‎clients/python/client/osparc/_http_client.py‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
from contextlib import suppress
12
from datetime import datetime
23
from email.utils import parsedate_to_datetime
34
from typing import Any, Awaitable, Callable, Dict, Optional, Set
@@ -93,21 +94,20 @@ async def get(self, *args, **kwargs) -> httpx.Response:
9394

9495
def _wait_callback(self, retry_state: tenacity.RetryCallState) -> int:
9596
assert retry_state.outcome is not None
96-
response: httpx.Response = retry_state.outcome.exception().response
97-
if response.status_code in _RETRY_AFTER_STATUS_CODES:
98-
retry_after = response.headers.get("Retry-After")
99-
if retry_after is not None:
100-
try:
101-
next_try = parsedate_to_datetime(retry_after)
102-
return int(
103-
(next_try - datetime.now(tz=next_try.tzinfo)).total_seconds()
104-
)
105-
except (ValueError, TypeError):
106-
pass
107-
try:
108-
return int(retry_after)
109-
except ValueError:
110-
pass
97+
if retry_state.outcome and retry_state.outcome.exception():
98+
response: httpx.Response = retry_state.outcome.exception().response
99+
if response.status_code in _RETRY_AFTER_STATUS_CODES:
100+
retry_after = response.headers.get("Retry-After")
101+
if retry_after is not None:
102+
with suppress(ValueError, TypeError):
103+
next_try = parsedate_to_datetime(retry_after)
104+
return int(
105+
(
106+
next_try - datetime.now(tz=next_try.tzinfo)
107+
).total_seconds()
108+
)
109+
with suppress(ValueError):
110+
return int(retry_after)
111111
# https://urllib3.readthedocs.io/en/stable/reference/urllib3.util.html#utilities
112112
return self.configuration.retries.backoff_factor * (
113113
2**retry_state.attempt_number

‎clients/python/client/osparc/_solvers_api.py‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ class SolversApi(_SolversApi):
2222
"get_jobs_page",
2323
]
2424

25-
def __getattribute__(self, name: str) -> Any:
25+
def __getattr__(self, name: str) -> Any:
2626
if (name in SolversApi._dev_features) and (not dev_features_enabled()):
2727
raise NotImplementedError(f"SolversApi.{name} is still under development")
2828
return super().__getattribute__(name)
@@ -33,8 +33,7 @@ def __init__(self, api_client: Optional[ApiClient] = None):
3333
Args:
3434
api_client (ApiClient, optinal): osparc.ApiClient object
3535
"""
36-
self._super: _SolversApi = super()
37-
self._super.__init__(api_client)
36+
super().__init__(api_client)
3837
user: Optional[str] = self.api_client.configuration.username
3938
passwd: Optional[str] = self.api_client.configuration.password
4039
self._auth: Optional[httpx.BasicAuth] = (
@@ -43,10 +42,12 @@ def __init__(self, api_client: Optional[ApiClient] = None):
4342
else None
4443
)
4544

46-
def list_solver_ports(self, solver_key: str, version: str) -> List[SolverPort]:
47-
page: OnePageSolverPort = self._super.list_solver_ports(
48-
solver_key=solver_key, version=version
49-
) # type: ignore
45+
def list_solver_ports(
46+
self, solver_key: str, version: str, **kwargs
47+
) -> List[SolverPort]:
48+
page: OnePageSolverPort = super().list_solver_ports(
49+
solver_key=solver_key, version=version, **kwargs
50+
)
5051
return page.items if page.items else []
5152

5253
@dev_feature

‎clients/python/client/osparc/_studies_api.py‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,7 @@ def __init__(self, api_client: Optional[ApiClient] = None):
4545
Args:
4646
api_client (ApiClient, optinal): osparc.ApiClient object
4747
"""
48-
self._super: _StudiesApi = super()
49-
self._super.__init__(api_client)
48+
super().__init__(api_client)
5049
user: Optional[str] = self.api_client.configuration.username
5150
passwd: Optional[str] = self.api_client.configuration.password
5251
self._auth: Optional[httpx.BasicAuth] = (
@@ -55,7 +54,7 @@ def __init__(self, api_client: Optional[ApiClient] = None):
5554
else None
5655
)
5756

58-
def __getattribute__(self, name: str) -> Any:
57+
def __getattr__(self, name: str) -> Any:
5958
if (name in StudiesApi._dev_features) and (not dev_features_enabled()):
6059
raise NotImplementedError(f"StudiesApi.{name} is still under development")
6160
return super().__getattribute__(name)
@@ -68,10 +67,12 @@ def clone_study(self, study_id: str, **kwargs):
6867
kwargs = {**kwargs, **ParentProjectInfo().model_dump(exclude_none=True)}
6968
return super().clone_study(study_id, **kwargs)
7069

71-
def studies(self) -> PaginationGenerator:
70+
def studies(self, **kwargs) -> PaginationGenerator:
7271
def _pagination_method():
73-
page_study = super(StudiesApi, self).list_studies(
74-
limit=_DEFAULT_PAGINATION_LIMIT, offset=_DEFAULT_PAGINATION_OFFSET
72+
page_study = self.list_studies(
73+
limit=_DEFAULT_PAGINATION_LIMIT,
74+
offset=_DEFAULT_PAGINATION_OFFSET,
75+
**kwargs,
7576
)
7677
assert isinstance(page_study, PageStudy) # nosec
7778
return page_study

‎clients/python/mypy.ini‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Global options
2+
[mypy]
3+
check_untyped_defs = True
4+
disallow_any_generics = False
5+
# disallow_untyped_defs: if True, it enforces things like `def __init__(self) -> CLASSNAME` or `def test_() -> None` which does not worth the effort
6+
disallow_untyped_defs = False
7+
follow_imports = silent
8+
# ignore_missing_imports: removes all the missing imports stuff from external libraries which is annoying to the least
9+
ignore_missing_imports = True
10+
namespace_packages = True
11+
no_implicit_reexport = True
12+
plugins = pydantic.mypy, sqlalchemy.ext.mypy.plugin
13+
python_version = 3.10
14+
show_column_numbers = True
15+
show_error_context = False
16+
strict_optional = False
17+
warn_redundant_casts = True
18+
warn_return_any = True
19+
warn_unused_configs = True
20+
warn_unused_ignores = True
21+
22+
# SEE https://docs.pydantic.dev/mypy_plugin/#plugin-settings
23+
[pydantic-mypy]
24+
init_forbid_extra = True
25+
init_typed = True
26+
warn_required_dynamic_aliases = True
27+
warn_untyped_fields = True

‎scripts/mypy.bash‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/bin/bash
2+
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3+
4+
set -o errexit
5+
set -o nounset
6+
set -o pipefail
7+
IFS=$'\n\t'
8+
9+
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
10+
IMAGE_NAME="local/scripts-$(basename "$0"):latest"
11+
WORKDIR="$(pwd)"
12+
13+
DEFAULT_MYPY_CONFIG="$(git rev-parse --show-toplevel)/clients/python/mypy.ini"
14+
MYPY_CONFIG=$(realpath "${2:-${DEFAULT_MYPY_CONFIG}}")
15+
16+
build() {
17+
echo Building image "$IMAGE_NAME"
18+
#
19+
docker buildx build \
20+
--load \
21+
--quiet \
22+
--tag "$IMAGE_NAME" \
23+
"$SCRIPT_DIR/mypy"
24+
}
25+
26+
echo_requirements() {
27+
echo "Installed :"
28+
docker run \
29+
--interactive \
30+
--rm \
31+
--user="$(id --user "$USER")":"$(id --group "$USER")" \
32+
--entrypoint="uv" \
33+
"$IMAGE_NAME" \
34+
--no-cache-dir pip freeze
35+
}
36+
37+
run() {
38+
echo Using "$(docker run --rm "$IMAGE_NAME" --version)"
39+
echo Mypy config "${MYPY_CONFIG}"
40+
echo Mypying "$(realpath "$@")":
41+
#
42+
docker run \
43+
--rm \
44+
--volume="/etc/group:/etc/group:ro" \
45+
--volume="/etc/passwd:/etc/passwd:ro" \
46+
--user="$(id --user "$USER")":"$(id --group "$USER")" \
47+
--volume "$MYPY_CONFIG":/config/mypy.ini \
48+
--volume "$WORKDIR":/src \
49+
--workdir=/src \
50+
"$IMAGE_NAME" \
51+
"$@"
52+
}
53+
54+
# ----------------------------------------------------------------------
55+
# MAIN
56+
#
57+
# USAGE
58+
# ./scripts/mypy.bash --help
59+
build
60+
echo_requirements
61+
run "$@"
62+
echo "DONE"
63+
# ----------------------------------------------------------------------

‎scripts/mypy/Dockerfile‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# syntax=docker/dockerfile:1
2+
ARG PYTHON_VERSION="3.10.14"
3+
FROM python:${PYTHON_VERSION}-slim-bookworm AS base
4+
5+
# Sets utf-8 encoding for Python et al
6+
ENV LANG=C.UTF-8
7+
8+
# Turns off writing .pyc files; superfluous on an ephemeral container.
9+
ENV PYTHONDONTWRITEBYTECODE=1 \
10+
VIRTUAL_ENV=/home/scu/.venv
11+
12+
# Ensures that the python and pip executables used in the image will be
13+
# those from our virtualenv.
14+
ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
15+
16+
17+
# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement
18+
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
19+
pip install uv~=0.2
20+
21+
RUN \
22+
--mount=type=cache,mode=0755,target=/root/.cache/uv \
23+
--mount=type=bind,source=./requirements.txt,target=requirements.txt \
24+
uv venv "${VIRTUAL_ENV}" \
25+
&& uv pip install --upgrade pip wheel setuptools \
26+
&& uv pip install -r requirements.txt \
27+
&& uv pip list
28+
29+
ENTRYPOINT ["mypy", "--config-file", "/config/mypy.ini", "--warn-unused-configs"]

‎scripts/mypy/requirements.txt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
httpx
2+
mypy~=1.10.0 # SEE https://github.com/ITISFoundation/osparc-simcore/issues/4750
3+
pydantic[email]~=1.10 # plugin for primary lib: keep in sync. SEE https://docs.pydantic.dev/mypy_plugin/#plugin-settings
4+
sqlalchemy[mypy]~=1.4 # plugin for primary lib: keep in sync. SEE https://docs.sqlalchemy.org/en/14/orm/extensions/mypy.html#installation
5+
tenacity
6+
types-aiofiles
7+
types-attrs
8+
types-cachetools
9+
types-PyYAML
10+
types-redis
11+
types-requests
12+
types-setuptools
13+
types-ujson

0 commit comments

Comments
 (0)