diff --git a/.gitignore b/.gitignore index c26311b7..a8219e3d 100644 --- a/.gitignore +++ b/.gitignore @@ -89,8 +89,3 @@ tmp* *ignore* !.dockerignore !.gitignore - - -# api diffs -api/openapi-*-diff.json -api/openapi-dev.json diff --git a/.vscode/settings.template.json b/.vscode/settings.template.json index 503f68bb..04607576 100644 --- a/.vscode/settings.template.json +++ b/.vscode/settings.template.json @@ -12,5 +12,8 @@ }, "pylint.args": [ "--rcfile=clients/python/.pylintrc" + ], + "python.analysis.extraPaths": [ + "./clients/python/artifacts/client" ] } diff --git a/Makefile b/Makefile index 917c83dc..5e410310 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ PYTHON_DIR := $(CLIENTS_DIR)/python .vscode/%.json: .vscode/%.template.json - $(if $(wildcard $@), \ + -$(if $(wildcard $@), \ @echo "WARNING ##### $< is newer than $@ ####"; diff -uN $@ $<; false;,\ @echo "WARNING ##### $@ does not exist, cloning $< as $@ ############"; cp $< $@) @@ -27,14 +27,16 @@ info-envs: ## info on envs info-tools: ## info on tooling # Tooling --------------- - @echo ' make : $(shell make --version 2>&1 | head -n 1)' - @echo ' jq : $(shell jq --version)' @echo ' awk : $(shell awk -W version 2>&1 | head -n 1)' - @echo ' python : $(shell python3 --version)' - @echo ' uv : $(shell uv --version)' + @echo ' curl : $(shell curl --version | head -n 1)' @echo ' docker : $(shell docker --version)' @echo ' docker buildx : $(shell docker buildx version)' @echo ' docker compose : $(shell docker compose version)' + @echo ' jq : $(shell jq --version)' + @echo ' make : $(shell make --version 2>&1 | head -n 1)' + @echo ' python : $(shell python3 --version)' + @echo ' uv : $(shell uv --version)' + info-pip: ## info index versions @@ -50,7 +52,9 @@ info: info-api info-envs info-tools info-pip ## all infos .venv: .check-uv-installed - @uv venv $@ + @uv venv \ + --python 3.10 \ + $@ ## upgrading tools to latest version in $(shell python3 --version) @uv pip --quiet install --upgrade \ pip~=24.0 \ diff --git a/README.md b/README.md index 2f67993a..cc706efc 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ To get started using the tools developed in this repository have a look at the [Documentation](https://itisfoundation.github.io/osparc-simcore-clients). For more in depth knowledge concerning the development of the repository, take a look at + - [Release Notes](https://github.com/ITISFoundation/osparc-simcore-clients/releases) - [Development notes](#development-notes) @@ -27,6 +28,7 @@ See the different `clients//README.md` for the workflows for generatin 2. The [openapi-generator](https://github.com/ITISFoundation/openapi-generator)-tool. The exact docker image of this tool to use is specifies in `scripts/common.Makefile`. ## Code lifecycle + This link explains the lifecycle of the osparc client(s) (borrowed from https://www.ibm.com/docs/en/acvfc?topic=manager-product-lifecycle)

diff --git a/VERSIONING.md b/VERSIONING.md new file mode 100644 index 00000000..796dc2b4 --- /dev/null +++ b/VERSIONING.md @@ -0,0 +1,15 @@ +# Versioning Strategy + +- We follow **post-release versioning** during development. + - Format: `1.2.3.post0.devN` where `N` represents the number of commits since the last release (`1.2.3`). + - We opt for post-release versioning rather than pre-release to avoid making early decisions about the next release version. + +- Official releases follow the format `1.2.3`. + +- **Patch releases** (e.g., `1.2.4`) are used instead of post-releases like `1.2.3.postX`. + +- Releases are determined by **git tags**. SEE [Releases](https://github.com/ITISFoundation/osparc-simcore-clients/releases). + +- For more details, refer to the following: + - GitHub workflow for publishing: `.github/workflows/publish-python-client.yml` + - Version computation script: `scripts/compute_version.bash` diff --git a/api/.gitignore b/api/.gitignore new file mode 100644 index 00000000..5bd3d54c --- /dev/null +++ b/api/.gitignore @@ -0,0 +1,3 @@ +openapi-*-diff.json +openapi-deploy.json +openapi-master.json diff --git a/api/Makefile b/api/Makefile index d3f8dc2e..c9075f7f 100644 --- a/api/Makefile +++ b/api/Makefile @@ -1,20 +1,31 @@ include ../scripts/common.Makefile -.PHONY: openapi-diff-dev.json -openapi-dev-diff.json: ## Diffs against newer openapi-dev.json and checks backwards compatibility - $(SCRIPTS_DIR)/openapi-diff.bash \ - /specs/openapi.json \ - /specs/openapi-dev.json \ - --fail-on-incompatible \ - --json=/specs/$@ +define download_openapi + @curl -L $(1) -o $(2) || (echo "Failed to download $(2)" && exit 1) +endef +.PHONY: openapi-master.json +openapi-master.json: + $(call download_openapi, https://raw.githubusercontent.com/ITISFoundation/osparc-simcore-clients/refs/heads/master/api/openapi.json,$@) +.PHONY: openapi-deploy.json +openapi-deploy.json: + $(call download_openapi, https://api.osparc.io/api/v0/openapi.json,$@) -.PHONY: openapi-diff-dev.json + +define openapi_diff + $(SCRIPTS_DIR)/openapi-diff.bash $(1) /specs/openapi.json \ + --fail-on-incompatible \ + --json="/specs/$(2)" +endef + + +.PHONY: openapi-master-diff.json +openapi-master-diff.json: ## Diffs against newer openapi-dev.json and checks backwards compatibility + $(call openapi_diff, https://raw.githubusercontent.com/ITISFoundation/osparc-simcore-clients/refs/heads/master/api/openapi.json,$@) + + +.PHONY: openapi-deploy-diff.json openapi-deploy-diff.json: ## Diffs against newer openapi-dev.json and checks backwards compatibility - $(SCRIPTS_DIR)/openapi-diff.bash \ - https://api.osparc.io/api/v0/openapi.json \ - /specs/openapi.json \ - --fail-on-incompatible \ - --json=/specs/$@ + $(call openapi_diff, https://api.osparc.io/api/v0/openapi.json,$@) diff --git a/api/openapi.json b/api/openapi.json index de50157c..7965ae50 100644 --- a/api/openapi.json +++ b/api/openapi.json @@ -1,9 +1,9 @@ { "openapi": "3.0.2", "info": { - "title": "osparc.io web API (dev)", + "title": "osparc.io public API", "description": "osparc-simcore public API specifications", - "version": "0.6.0-dev" + "version": "0.7.0" }, "paths": { "/v0/meta": { @@ -223,7 +223,7 @@ "files" ], "summary": "List Files", - "description": "Lists all files stored in the system\n\nSEE get_files_page for a paginated version of this function", + "description": "Lists all files stored in the system\n\nSEE `get_files_page` for a paginated version of this function", "operationId": "list_files", "responses": { "200": { @@ -308,67 +308,6 @@ ] } }, - "/v0/files/page": { - "get": { - "tags": [ - "files" - ], - "summary": "Get Files Page", - "operationId": "get_files_page", - "parameters": [ - { - "required": false, - "schema": { - "type": "integer", - "maximum": 100, - "minimum": 1, - "title": "Limit", - "default": 50 - }, - "name": "limit", - "in": "query" - }, - { - "required": false, - "schema": { - "type": "integer", - "minimum": 0, - "title": "Offset", - "default": 0 - }, - "name": "offset", - "in": "query" - } - ], - "responses": { - "501": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Page_File_" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - }, - "security": [ - { - "HTTPBasic": [] - } - ] - } - }, "/v0/files/content": { "put": { "tags": [ @@ -1379,62 +1318,6 @@ ] } }, - "/v0/solvers/page": { - "get": { - "tags": [ - "solvers" - ], - "summary": "Get Solvers Page", - "operationId": "get_solvers_page", - "parameters": [ - { - "required": false, - "schema": { - "type": "integer", - "maximum": 100, - "minimum": 1, - "title": "Limit", - "default": 50 - }, - "name": "limit", - "in": "query" - }, - { - "required": false, - "schema": { - "type": "integer", - "minimum": 0, - "title": "Offset", - "default": 0 - }, - "name": "offset", - "in": "query" - } - ], - "responses": { - "501": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Page_Solver_" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, "/v0/solvers/releases": { "get": { "tags": [ @@ -1526,62 +1409,6 @@ ] } }, - "/v0/solvers/releases/page": { - "get": { - "tags": [ - "solvers" - ], - "summary": "Get Solvers Releases Page", - "operationId": "get_solvers_releases_page", - "parameters": [ - { - "required": false, - "schema": { - "type": "integer", - "maximum": 100, - "minimum": 1, - "title": "Limit", - "default": 50 - }, - "name": "limit", - "in": "query" - }, - { - "required": false, - "schema": { - "type": "integer", - "minimum": 0, - "title": "Offset", - "default": 0 - }, - "name": "offset", - "in": "query" - } - ], - "responses": { - "501": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Page_Solver_" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, "/v0/solvers/{solver_key}/latest": { "get": { "tags": [ @@ -1804,72 +1631,6 @@ ] } }, - "/v0/solvers/{solver_key}/releases/page": { - "get": { - "tags": [ - "solvers" - ], - "summary": "Get Solver Releases Page", - "operationId": "get_solver_releases_page", - "parameters": [ - { - "required": true, - "schema": { - "type": "string", - "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", - "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" - }, - { - "required": false, - "schema": { - "type": "integer", - "maximum": 100, - "minimum": 1, - "title": "Limit", - "default": 50 - }, - "name": "limit", - "in": "query" - }, - { - "required": false, - "schema": { - "type": "integer", - "minimum": 0, - "title": "Offset", - "default": 0 - }, - "name": "offset", - "in": "query" - } - ], - "responses": { - "501": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Page_Solver_" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, "/v0/solvers/{solver_key}/releases/{version}": { "get": { "tags": [ @@ -2114,6 +1875,7 @@ "solvers" ], "summary": "Get Solver Pricing Plan", + "description": "Gets solver pricing plan\n\nNew in *version 0.7*", "operationId": "get_solver_pricing_plan", "parameters": [ { @@ -2232,7 +1994,7 @@ "solvers" ], "summary": "List Jobs", - "description": "List of jobs in a specific released solver (limited to 20 jobs)\n\nSEE get_jobs_page for paginated version of this function", + "description": "List of jobs in a specific released solver (limited to 20 jobs)\n\n- DEPRECATION: This implementation and returned values are deprecated and the will be replaced by that of get_jobs_page\n- SEE `get_jobs_page` for paginated version of this function", "operationId": "list_jobs", "parameters": [ { @@ -2669,7 +2431,7 @@ "solvers" ], "summary": "Delete Job", - "description": "Deletes an existing solver job\n\nNew in *version 0.5*", + "description": "Deletes an existing solver job\n\nNew in *version 0.7*", "operationId": "delete_job", "parameters": [ { @@ -2801,7 +2563,7 @@ "solvers" ], "summary": "Start Job", - "description": "Starts job job_id created with the solver solver_key:version\n\nNew in *version 0.4.3*: cluster_id\nNew in *version 0.6.0*: This endpoint responds with a 202 when successfully starting a computation", + "description": "Starts job job_id created with the solver solver_key:version\n\nAdded in *version 0.4.3*: query parameter `cluster_id`\nAdded in *version 0.6*: responds with a 202 when successfully starting a computation", "operationId": "start_job", "parameters": [ { @@ -3246,7 +3008,7 @@ "solvers" ], "summary": "Get Job Custom Metadata", - "description": "Gets custom metadata from a job\n\nNew in *version 0.5*", + "description": "Gets custom metadata from a job\n\nNew in *version 0.7*", "operationId": "get_job_custom_metadata", "parameters": [ { @@ -3373,7 +3135,7 @@ "solvers" ], "summary": "Replace Job Custom Metadata", - "description": "Updates custom metadata from a job\n\nNew in *version 0.5*", + "description": "Updates custom metadata from a job\n\nNew in *version 0.7*", "operationId": "replace_job_custom_metadata", "parameters": [ { @@ -3512,7 +3274,7 @@ "solvers" ], "summary": "Get Jobs Page", - "description": "List of jobs on a specific released solver (includes pagination)", + "description": "List of jobs on a specific released solver (includes pagination)\n\nNew in *version 0.7*", "operationId": "get_jobs_page", "parameters": [ { @@ -3802,7 +3564,7 @@ "solvers" ], "summary": "Get Job Output Logfile", - "description": "Special extra output with persistent logs file for the solver run.\n\nNOTE: this is not a log stream but a predefined output that is only\navailable after the job is done.\n\nNew in *version 0.4.0*", + "description": "Special extra output with persistent logs file for the solver run.\n\n**NOTE**: this is not a log stream but a predefined output that is only\navailable after the job is done.\n\nNew in *version 0.4.0*", "operationId": "get_job_output_logfile", "parameters": [ { @@ -3939,6 +3701,7 @@ "solvers" ], "summary": "Get Job Wallet", + "description": "Get job wallet\n\nNew in *version 0.7*", "operationId": "get_job_wallet", "parameters": [ { @@ -4077,6 +3840,7 @@ "solvers" ], "summary": "Get Job Pricing Unit", + "description": "Get job pricing unit\n\nNew in *version 0.7*", "operationId": "get_job_pricing_unit", "parameters": [ { @@ -4240,11 +4004,20 @@ ], "responses": { "200": { - "description": "Successful Response", + "description": "Returns a JobLog or an ErrorGet", "content": { "application/x-ndjson": { "schema": { - "type": "string" + "anyOf": [ + { + "$ref": "#/components/schemas/JobLog" + }, + { + "$ref": "#/components/schemas/ErrorGet" + } + ], + "type": "string", + "title": "Response 200 Get Log Stream V0 Solvers Solver Key Releases Version Jobs Job Id Logstream Get" } } } @@ -4586,70 +4359,6 @@ } }, "/v0/studies/{study_id}/jobs": { - "get": { - "tags": [ - "studies" - ], - "summary": "List Study Jobs", - "operationId": "list_study_jobs", - "parameters": [ - { - "required": true, - "schema": { - "type": "string", - "format": "uuid", - "title": "Study Id" - }, - "name": "study_id", - "in": "path" - }, - { - "required": false, - "schema": { - "type": "integer", - "maximum": 100, - "minimum": 1, - "title": "Limit", - "default": 50 - }, - "name": "limit", - "in": "query" - }, - { - "required": false, - "schema": { - "type": "integer", - "minimum": 0, - "title": "Offset", - "default": 0 - }, - "name": "offset", - "in": "query" - } - ], - "responses": { - "501": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Page_Job_" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - }, "post": { "tags": [ "studies" @@ -4739,57 +4448,6 @@ } }, "/v0/studies/{study_id}/jobs/{job_id}": { - "get": { - "tags": [ - "studies" - ], - "summary": "Get Study Job", - "operationId": "get_study_job", - "parameters": [ - { - "required": true, - "schema": { - "type": "string", - "format": "uuid", - "title": "Study Id" - }, - "name": "study_id", - "in": "path" - }, - { - "required": true, - "schema": { - "type": "string", - "format": "uuid", - "title": "Job Id" - }, - "name": "job_id", - "in": "path" - } - ], - "responses": { - "501": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Job" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - }, "delete": { "tags": [ "studies" @@ -4857,7 +4515,7 @@ "studies" ], "summary": "Start Study Job", - "description": "New in *version 0.6.0*: This endpoint responds with a 202 when successfully starting a computation", + "description": "Changed in *version 0.6.0*: Now responds with a 202 when successfully starting a computation", "operationId": "start_study_job", "parameters": [ { @@ -5248,7 +4906,7 @@ "studies" ], "summary": "Get Study Job Custom Metadata", - "description": "Gets custom metadata from a job", + "description": "Get custom metadata from a study's job\n\nNew in *version 0.7*", "operationId": "get_study_job_custom_metadata", "parameters": [ { @@ -5305,7 +4963,7 @@ "studies" ], "summary": "Replace Study Job Custom Metadata", - "description": "Changes job's custom metadata", + "description": "Changes custom metadata of a study's job\n\nNew in *version 0.7*", "operationId": "replace_study_job_custom_metadata", "parameters": [ { @@ -5374,6 +5032,7 @@ "wallets" ], "summary": "Get Default Wallet", + "description": "Get default wallet\n\nNew in *version 0.7*", "operationId": "get_default_wallet", "responses": { "200": { @@ -5470,6 +5129,7 @@ "wallets" ], "summary": "Get Wallet", + "description": "Get wallet\n\nNew in *version 0.7*", "operationId": "get_wallet", "parameters": [ { @@ -5587,6 +5247,7 @@ "credits" ], "summary": "Get Credits Price", + "description": "New in *version 0.6.0*", "operationId": "get_credits_price", "responses": { "200": { @@ -5996,6 +5657,46 @@ } } }, + "JobLog": { + "properties": { + "job_id": { + "type": "string", + "format": "uuid", + "title": "Job Id" + }, + "node_id": { + "type": "string", + "format": "uuid", + "title": "Node Id" + }, + "log_level": { + "type": "integer", + "title": "Log Level" + }, + "messages": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Messages" + } + }, + "type": "object", + "required": [ + "job_id", + "log_level", + "messages" + ], + "title": "JobLog", + "example": { + "job_id": "145beae4-a3a8-4fde-adbb-4e8257c2c083", + "node_id": "3742215e-6756-48d2-8b73-4d043065309f", + "log_level": 10, + "messages": [ + "PROGRESS: 5/10" + ] + } + }, "JobLogsMap": { "properties": { "log_links": { @@ -6413,41 +6114,6 @@ ], "title": "Page[Job]" }, - "Page_Solver_": { - "properties": { - "items": { - "items": { - "$ref": "#/components/schemas/Solver" - }, - "type": "array", - "title": "Items" - }, - "total": { - "type": "integer", - "minimum": 0, - "title": "Total" - }, - "limit": { - "type": "integer", - "minimum": 1, - "title": "Limit" - }, - "offset": { - "type": "integer", - "minimum": 0, - "title": "Offset" - }, - "links": { - "$ref": "#/components/schemas/Links" - } - }, - "type": "object", - "required": [ - "items", - "links" - ], - "title": "Page[Solver]" - }, "Page_Study_": { "properties": { "items": { @@ -6949,6 +6615,8 @@ }, "name": { "type": "string", + "maxLength": 100, + "minLength": 1, "title": "Name" }, "description": { diff --git a/clients/python/Makefile b/clients/python/Makefile index 28e64b4d..5c79dd89 100644 --- a/clients/python/Makefile +++ b/clients/python/Makefile @@ -15,15 +15,31 @@ ADDITIONAL_PROPS := \ ADDITIONAL_PROPS := $(foreach prop,$(ADDITIONAL_PROPS),$(strip $(prop))) VERSION_FILE := $(CLIENTS_PYTHON_DIR)/VERSION -.PHONY: generate_version -VERSION: - @bash $(SCRIPTS_DIR)/generate_version.bash > $(VERSION_FILE) +.PHONY: VERSION +VERSION: ## Computes and dumps current client version + @printf "$$(bash $(SCRIPTS_DIR)/compute_version.bash)" > $(VERSION_FILE) || (printf "Failed to compute version\n" && exit 1) .PHONY: artifacts_dir artifacts_dir: -mkdir --parents $(ARTIFACTS_DIR) +.PHONY: python-client-generator-help +python-client-generator-help: ## help on client-api generator + # generate help + @docker run --rm $(OPENAPI_GENERATOR_IMAGE) help generate + + +.PHONY: _postprocess_osparc_client_artifacts +_postprocess_osparc_client_artifacts: + black $(ARTIFACTS_DIR)/client/*.py + @-rm -f $(ARTIFACTS_DIR)/client/README.md + @echo "Please visit our [website](https://itisfoundation.github.io/osparc-simcore-clients/#/) for documentation" > $(ARTIFACTS_DIR)/client/README.md + @-rm -rf $(CLIENTS_PYTHON_DIR)/src/osparc/data/ + @-mkdir $(CLIENTS_PYTHON_DIR)/src/osparc/data/ + @cp $(REPO_ROOT)/api/openapi.json $(CLIENTS_PYTHON_DIR)/src/osparc/data/ + + # Generation of Python client .PHONY: python-client python-client: VERSION validate-api-specification artifacts_dir ## auto-generate python client @@ -41,7 +57,7 @@ python-client: VERSION validate-api-specification artifacts_dir ## auto-generate --additional-properties=packageVersion=$(VERSION),$(subst $(space),$(comma),$(strip $(ADDITIONAL_PROPS))) \ --package-name=osparc_client \ --release-note="Updated to $(VERSION)" - $(MAKE) postprocess-build + $(MAKE) _postprocess_osparc_client_artifacts .PHONY: python-client-from-templates python-client-from-templates: VERSION validate-api-specification artifacts_dir ## generate python client using templates in a specified directory (usage: 'make python-client-from-templates -e TEMPLATE_DIR=path/to/templates') @@ -63,14 +79,11 @@ python-client-from-templates: VERSION validate-api-specification artifacts_dir # --package-name=osparc_client \ --release-note="Updated to $(VERSION)" \ --template-dir=/tmp/python_templates - $(MAKE) postprocess-build + $(MAKE) _postprocess_osparc_client_artifacts + -.PHONY: generator-help -generator-help: ## help on client-api generator - # generate help - @docker run --rm $(OPENAPI_GENERATOR_IMAGE) help generate -## Tools for "postprocessing" generated documentation -------------------------------------------- +## DOCUMENTATION -------------------------------------------- ARTIFACTS_DOCS := artifacts/docs @@ -95,7 +108,7 @@ _check_venv_active: # checking whether virtual environment was activated @python3 -c "import sys; assert sys.base_prefix!=sys.prefix" - +# (0) Environment --------------------- .PHONY: .install-dev-reqs .install-dev-reqs: _check_venv_active @@ -120,6 +133,7 @@ install-doc: _check_venv_active .install-dev-reqs ## install packages for genera uv pip install -r $(CLIENTS_PYTHON_DIR)/requirements/doc.txt +# (1) Linters --------------------- .PHONY: pylint pylint: _check_venv_active ## runs linter (only to check errors. SEE .pylintrc enabled) @@ -130,9 +144,10 @@ mypy: $(SCRIPTS_DIR)/mypy.bash $(CLIENTS_PYTHON_DIR)/mypy.ini ## runs mypy pytho @$(SCRIPTS_DIR)/mypy.bash src/osparc +# (2) Testing --------------------- -.PHONY: test-dev -test-dev: _check_venv_active ## runs tests during development +.PHONY: tests +tests: _check_venv_active ## runs tests during development # runs tests for development (e.g w/ pdb) python -m pytest \ -vv \ @@ -142,7 +157,7 @@ test-dev: _check_venv_active ## runs tests during development --pdb \ $(CLIENTS_PYTHON_DIR)/test/test_osparc - +# (3) Packaging ------------------- .PHONY: dist dist: artifacts_dir ## builds distribution wheel for `osparc_client` and `osparc` packages -> $(ARTIFACTS_DIR)/dist @@ -162,15 +177,6 @@ dist: artifacts_dir ## builds distribution wheel for `osparc_client` and `osparc -.PHONY: postprocess-build -postprocess-build: - black $(ARTIFACTS_DIR)/client/*.py - @-rm -f $(ARTIFACTS_DIR)/client/README.md - @echo "Please visit our [website](https://itisfoundation.github.io/osparc-simcore-clients/#/) for documentation" > $(ARTIFACTS_DIR)/client/README.md - @-rm -rf $(CLIENTS_PYTHON_DIR)/src/osparc/data/ - @-mkdir $(CLIENTS_PYTHON_DIR)/src/osparc/data/ - @cp $(REPO_ROOT)/api/openapi.json $(CLIENTS_PYTHON_DIR)/src/osparc/data/ - ## e2e TEST --------------------------------------------------------------------------------- diff --git a/clients/python/README.md b/clients/python/README.md index 2324d713..757b0633 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -20,7 +20,7 @@ Changes to the client should not be performed within the generated python code, Tests are located in `PYDIR/test_osparc` and `PYDIR/test_osparc_client`. They can be run by executing the following commands from PYDIR: ```bash make install-test -make test-dev +make tests ``` after the client has been generated (see the [workflow section](#workflow)). These tests should test the user-facing API. Jupyter notebooks in the PYDIR/docs folder are automatically run as part of the tests via `test_notebooks.py`. Hence, when writing a tutorial jupyter script, also think of it as a test. For this reason, any import of `osparc` in the notebook should be "guarded" by a `if importlib.util.find_spec('osparc') is None:` statement (see the [BasicTutorial](./docs/BasicTutorial.ipynb)) diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 4a5e0b3c..ec1555fe 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -29,9 +29,7 @@ warn_untyped_fields = true [tool.pytest.ini_options] # SEE https://docs.pytest.org/en/7.1.x/reference/customize.html#pyproject-toml -testpaths = [ - "test", -] +testpaths = ["test"] # SEE https://pytest-asyncio.readthedocs.io/en/stable/concepts.html#test-discovery-modes asyncio_mode = "auto" required_plugins = "pytest-asyncio" diff --git a/clients/python/setup.py b/clients/python/setup.py index 23ce764c..28e09da5 100644 --- a/clients/python/setup.py +++ b/clients/python/setup.py @@ -19,7 +19,7 @@ assert VERSION_FILE.is_file(), "Did you forget `make VERSION`?" NAME = "osparc" -VERSION = VERSION_FILE.read_text() +VERSION = VERSION_FILE.read_text().strip() REQUIRES = [ "httpx", diff --git a/clients/python/src/osparc/__init__.py b/clients/python/src/osparc/__init__.py index 7d8f94b0..934e60a6 100644 --- a/clients/python/src/osparc/__init__.py +++ b/clients/python/src/osparc/__init__.py @@ -1,117 +1,11 @@ -import warnings -from platform import python_version -from typing import List, Tuple - import nest_asyncio -from osparc_client import ( - ApiException, - ApiKeyError, - ApiTypeError, - ApiValueError, - BodyUploadFileV0FilesContentPut, - Configuration, - CreditsApi, - ErrorGet, - File, - Groups, - HTTPValidationError, - Job, - JobInputs, - JobOutputs, - JobStatus, - Meta, - MetaApi, - OnePageSolverPort, - OpenApiException, - Profile, - ProfileUpdate, - Solver, - SolverPort, - UserRoleEnum, - UsersApi, - UsersGroup, - ValidationError, - __version__, -) -from osparc_client import RunningState as TaskStates -from packaging.version import Version - -from ._api_client import ApiClient -from ._exceptions import RequestError, VisibleDeprecationWarning -from ._files_api import FilesApi -from ._info import openapi -from ._solvers_api import SolversApi -from ._studies_api import StudiesApi -from ._utils import dev_features_enabled - -_PYTHON_VERSION_RETIRED = Version("3.8.0") -_PYTHON_VERSION_DEPRECATED = Version("3.8.0") -assert _PYTHON_VERSION_RETIRED <= _PYTHON_VERSION_DEPRECATED # nosec +from ._version import __version__ as __version__ -if Version(python_version()) < _PYTHON_VERSION_RETIRED: - error_msg: str = ( - f"Python version {python_version()} is retired for this version of osparc. " - f"Please use Python version {_PYTHON_VERSION_DEPRECATED}." - ) - raise RuntimeError(error_msg) - -if Version(python_version()) < _PYTHON_VERSION_DEPRECATED: - warning_msg: str = ( - f"Python {python_version()} is deprecated. " - "Please upgrade to " - f"Python version >= {_PYTHON_VERSION_DEPRECATED}." - ) - warnings.warn(warning_msg, VisibleDeprecationWarning) +# APIs +from .exceptions import * # noqa: F403 +from .models import * # noqa: F403 +from .api import * # noqa: F403 +from ._info import openapi as openapi nest_asyncio.apply() # allow to run coroutines via asyncio.run(coro) - -dev_features: List[str] = [] -if dev_features_enabled(): - dev_features = [ - "JobMetadata", - "JobMetadataUpdate", - "Links", - "OnePageStudyPort", - "PaginationGenerator", - "Study", - "StudyPort", - ] - -__all__: Tuple[str, ...] = tuple(dev_features) + ( - "__version__", - "ApiClient", - "ApiException", - "ApiKeyError", - "ApiTypeError", - "ApiValueError", - "BodyUploadFileV0FilesContentPut", - "Configuration", - "CreditsApi", - "ErrorGet", - "File", - "FilesApi", - "Groups", - "HTTPValidationError", - "Job", - "JobInputs", - "JobOutputs", - "JobStatus", - "Meta", - "MetaApi", - "OnePageSolverPort", - "openapi", - "OpenApiException", - "Profile", - "ProfileUpdate", - "RequestError", - "Solver", - "SolverPort", - "SolversApi", - "StudiesApi", - "TaskStates", - "UserRoleEnum", - "UsersApi", - "UsersGroup", - "ValidationError", -) # type: ignore diff --git a/clients/python/src/osparc/_api_client.py b/clients/python/src/osparc/_api_client.py index 574506f1..70cc5b40 100644 --- a/clients/python/src/osparc/_api_client.py +++ b/clients/python/src/osparc/_api_client.py @@ -1,10 +1,12 @@ +# wraps osparc_client.api_client + from typing import Optional -from osparc_client import ApiClient as _ApiClient +from osparc_client.api_client import ApiClient as _ApiClient from osparc_client import Configuration from pydantic import ValidationError -from ._settings import ConfigurationModel +from ._settings import ConfigurationEnvVars class ApiClient(_ApiClient): @@ -18,7 +20,7 @@ def __init__( ): if configuration is None: try: - env_vars = ConfigurationModel() + env_vars = ConfigurationEnvVars() configuration = Configuration( host=f"{env_vars.OSPARC_API_HOST}".rstrip( "/" @@ -28,7 +30,7 @@ def __init__( ) except ValidationError as exc: raise RuntimeError( - f"Could not initialize configuration from environment (expected {ConfigurationModel.model_fields_set}). " + f"Could not initialize configuration from environment (expected {ConfigurationEnvVars.model_fields_set}). " "If your osparc host, key and secret are not exposed as " "environment variables you must construct the " "osparc.Configuration object explicitly" diff --git a/clients/python/src/osparc/_files_api.py b/clients/python/src/osparc/_api_files_api.py similarity index 97% rename from clients/python/src/osparc/_files_api.py rename to clients/python/src/osparc/_api_files_api.py index 483f9f5a..88bdea55 100644 --- a/clients/python/src/osparc/_files_api.py +++ b/clients/python/src/osparc/_api_files_api.py @@ -1,3 +1,5 @@ +# wraps osparc_client.api.files_api + import asyncio import json import logging @@ -10,26 +12,28 @@ import httpx from httpx import Response -from osparc_client import ( - BodyAbortMultipartUploadV0FilesFileIdAbortPost, - BodyCompleteMultipartUploadV0FilesFileIdCompletePost, - ClientFile, - ClientFileUploadData, -) -from osparc_client import FilesApi as _FilesApi -from osparc_client import FileUploadCompletionBody, FileUploadData, UploadedPart +from osparc_client.api.files_api import FilesApi as _FilesApi from tqdm.asyncio import tqdm from tqdm.contrib.logging import logging_redirect_tqdm -from . import File from ._api_client import ApiClient from ._http_client import AsyncHttpClient +from .models import ( + BodyAbortMultipartUploadV0FilesFileIdAbortPost, + BodyCompleteMultipartUploadV0FilesFileIdCompletePost, + ClientFile, + ClientFileUploadData, + File, + FileUploadCompletionBody, + FileUploadData, + UploadedPart, +) from ._utils import ( DEFAULT_TIMEOUT_SECONDS, PaginationGenerator, compute_sha256, - file_chunk_generator, dev_features_enabled, + file_chunk_generator, ) _logger = logging.getLogger(__name__) diff --git a/clients/python/src/osparc/_solvers_api.py b/clients/python/src/osparc/_api_solvers_api.py similarity index 95% rename from clients/python/src/osparc/_solvers_api.py rename to clients/python/src/osparc/_api_solvers_api.py index 39ee5d15..7b66d854 100644 --- a/clients/python/src/osparc/_solvers_api.py +++ b/clients/python/src/osparc/_api_solvers_api.py @@ -1,8 +1,10 @@ +# Wraps osparc_client.api.solvers_api + from typing import Any, List, Optional import httpx -from osparc_client import JobInputs, OnePageSolverPort, SolverPort -from osparc_client import SolversApi as _SolversApi +from osparc_client.api.solvers_api import SolversApi as _SolversApi +from .models import JobInputs, OnePageSolverPort, SolverPort from ._api_client import ApiClient from ._settings import ParentProjectInfo diff --git a/clients/python/src/osparc/_studies_api.py b/clients/python/src/osparc/_api_studies_api.py similarity index 96% rename from clients/python/src/osparc/_studies_api.py rename to clients/python/src/osparc/_api_studies_api.py index 7f412786..bb78bd55 100644 --- a/clients/python/src/osparc/_studies_api.py +++ b/clients/python/src/osparc/_api_studies_api.py @@ -1,3 +1,5 @@ +# Wraps osparc_client.api.studies_api + import asyncio import logging from pathlib import Path @@ -5,8 +7,8 @@ from typing import Any, Optional import httpx -from osparc_client import JobInputs, JobLogsMap, PageStudy -from osparc_client import StudiesApi as _StudiesApi +from .models import JobInputs, JobLogsMap, PageStudy +from osparc_client.api.studies_api import StudiesApi as _StudiesApi from tqdm.asyncio import tqdm_asyncio from ._api_client import ApiClient diff --git a/clients/python/src/osparc/_exceptions.py b/clients/python/src/osparc/_exceptions.py index 869a83e2..938a2ad8 100644 --- a/clients/python/src/osparc/_exceptions.py +++ b/clients/python/src/osparc/_exceptions.py @@ -1,7 +1,11 @@ +# pylint: disable=unused-import + from functools import wraps from httpx import HTTPStatusError -from osparc_client import ApiException + + +from osparc_client.exceptions import ApiException as ApiException class VisibleDeprecationWarning(UserWarning): diff --git a/clients/python/src/osparc/_settings.py b/clients/python/src/osparc/_settings.py index 9d2e0a54..3e2d5c5e 100644 --- a/clients/python/src/osparc/_settings.py +++ b/clients/python/src/osparc/_settings.py @@ -25,7 +25,7 @@ def _validate_uuids(cls, v: Optional[str]) -> Optional[str]: return v -class ConfigurationModel(BaseSettings): +class ConfigurationEnvVars(BaseSettings): """Model for capturing env vars which should go into the Configuration""" # Service side: https://github.com/ITISFoundation/osparc-simcore/pull/5966 diff --git a/clients/python/src/osparc/_utils.py b/clients/python/src/osparc/_utils.py index 77ebc78b..5a012826 100644 --- a/clients/python/src/osparc/_utils.py +++ b/clients/python/src/osparc/_utils.py @@ -12,7 +12,6 @@ Job, PageFile, PageJob, - PageSolver, PageStudy, Solver, Study, @@ -29,7 +28,7 @@ DEFAULT_TIMEOUT_SECONDS: int = 30 * 60 -Page = Union[PageJob, PageFile, PageSolver, PageStudy] +Page = Union[PageJob, PageFile, PageStudy] T = TypeVar("T", Job, File, Solver, Study) @@ -128,9 +127,9 @@ def dev_features_enabled() -> bool: def dev_feature(func: Callable): @wraps(func) - def wrapper(*args, **kwargs): + def _wrapper(*args, **kwargs): if not dev_features_enabled(): raise NotImplementedError(f"{func.__name__} is still under development") return func(*args, **kwargs) - return wrapper + return _wrapper diff --git a/clients/python/src/osparc/_version.py b/clients/python/src/osparc/_version.py new file mode 100644 index 00000000..7e0021c0 --- /dev/null +++ b/clients/python/src/osparc/_version.py @@ -0,0 +1,26 @@ +import warnings +from platform import python_version +from packaging.version import Version +from ._exceptions import VisibleDeprecationWarning + +from osparc_client import __version__ as __version__ + + +_PYTHON_VERSION_RETIRED = Version("3.8.0") +_PYTHON_VERSION_DEPRECATED = Version("3.8.0") +assert _PYTHON_VERSION_RETIRED <= _PYTHON_VERSION_DEPRECATED # nosec + +if Version(python_version()) < _PYTHON_VERSION_RETIRED: + error_msg: str = ( + f"Python version {python_version()} is retired for this version of osparc. " + f"Please use Python version {_PYTHON_VERSION_DEPRECATED}." + ) + raise RuntimeError(error_msg) + +if Version(python_version()) < _PYTHON_VERSION_DEPRECATED: + warning_msg: str = ( + f"Python {python_version()} is deprecated. " + "Please upgrade to " + f"Python version >= {_PYTHON_VERSION_DEPRECATED}." + ) + warnings.warn(warning_msg, VisibleDeprecationWarning) diff --git a/clients/python/src/osparc/api.py b/clients/python/src/osparc/api.py index 7db91030..24a07c0d 100644 --- a/clients/python/src/osparc/api.py +++ b/clients/python/src/osparc/api.py @@ -1,16 +1,19 @@ -import warnings -from typing import Final, Tuple +# pylint: disable=unused-import -from osparc_client.api import FilesApi, MetaApi, SolversApi, UsersApi +# NOTE: Now includes Configuration, ApiClient +# NOTE: this is an interface. Keep it clean! -from ._exceptions import VisibleDeprecationWarning -warning_msg: Final[str] = ( - "osparc.api has been deprecated. Instead functionality within this module " - "should be imported directly from osparc. I.e. please do 'from osparc import " - "' instead of 'from osparc.api import '" -) -warnings.warn(warning_msg, VisibleDeprecationWarning) +from osparc_client.configuration import Configuration as Configuration +from ._api_client import ApiClient as ApiClient -__all__: Tuple[str, ...] = ("FilesApi", "MetaApi", "SolversApi", "UsersApi") +from osparc_client.api.credits_api import CreditsApi as CreditsApi +from osparc_client.api.meta_api import MetaApi as MetaApi +from osparc_client.api.users_api import UsersApi as UsersApi +from osparc_client.api.wallets_api import WalletsApi as WalletsApi + + +from ._api_solvers_api import SolversApi as SolversApi +from ._api_studies_api import StudiesApi as StudiesApi +from ._api_files_api import FilesApi as FilesApi diff --git a/clients/python/src/osparc/exceptions.py b/clients/python/src/osparc/exceptions.py new file mode 100644 index 00000000..254336a8 --- /dev/null +++ b/clients/python/src/osparc/exceptions.py @@ -0,0 +1,9 @@ +# pylint: disable=unused-import + +from osparc_client.exceptions import OpenApiException as OpenApiException +from osparc_client.exceptions import ApiTypeError as ApiTypeError +from osparc_client.exceptions import ApiValueError as ApiValueError +from osparc_client.exceptions import ApiKeyError as ApiKeyError +from osparc_client.exceptions import ApiException as ApiException + +from ._exceptions import RequestError as RequestError diff --git a/clients/python/src/osparc/models.py b/clients/python/src/osparc/models.py new file mode 100644 index 00000000..9c4aa508 --- /dev/null +++ b/clients/python/src/osparc/models.py @@ -0,0 +1,80 @@ +# wraps osparc_client.models +# +# TIP: +# search "import (\w+)" +# replace "import $1 as $1" +# +# NOTE: this is an interface. Keep it clean! + +from osparc_client.models.body_abort_multipart_upload_v0_files_file_id_abort_post import ( + BodyAbortMultipartUploadV0FilesFileIdAbortPost as BodyAbortMultipartUploadV0FilesFileIdAbortPost, +) +from osparc_client.models.body_complete_multipart_upload_v0_files_file_id_complete_post import ( + BodyCompleteMultipartUploadV0FilesFileIdCompletePost as BodyCompleteMultipartUploadV0FilesFileIdCompletePost, +) +from osparc_client.models.body_upload_file_v0_files_content_put import ( + BodyUploadFileV0FilesContentPut as BodyUploadFileV0FilesContentPut, +) +from osparc_client.models.client_file import ClientFile as ClientFile +from osparc_client.models.client_file_upload_data import ( + ClientFileUploadData as ClientFileUploadData, +) +from osparc_client.models.error_get import ErrorGet as ErrorGet +from osparc_client.models.file import File as File +from osparc_client.models.file_upload_completion_body import ( + FileUploadCompletionBody as FileUploadCompletionBody, +) +from osparc_client.models.file_upload_data import FileUploadData as FileUploadData +from osparc_client.models.get_credit_price import GetCreditPrice as GetCreditPrice +from osparc_client.models.groups import Groups as Groups +from osparc_client.models.http_validation_error import ( + HTTPValidationError as HTTPValidationError, +) +from osparc_client.models.job import Job as Job +from osparc_client.models.job_inputs import JobInputs as JobInputs +from osparc_client.models.job_logs_map import JobLogsMap as JobLogsMap +from osparc_client.models.job_metadata import JobMetadata as JobMetadata +from osparc_client.models.job_metadata_update import ( + JobMetadataUpdate as JobMetadataUpdate, +) +from osparc_client.models.job_outputs import JobOutputs as JobOutputs +from osparc_client.models.job_status import JobStatus as JobStatus +from osparc_client.models.links import Links as Links +from osparc_client.models.log_link import LogLink as LogLink +from osparc_client.models.meta import Meta as Meta +from osparc_client.models.one_page_solver_port import ( + OnePageSolverPort as OnePageSolverPort, +) +from osparc_client.models.one_page_study_port import ( + OnePageStudyPort as OnePageStudyPort, +) +from osparc_client.models.page_file import PageFile as PageFile +from osparc_client.models.page_job import PageJob as PageJob +from osparc_client.models.page_study import PageStudy as PageStudy +from osparc_client.models.pricing_plan_classification import ( + PricingPlanClassification as PricingPlanClassification, +) +from osparc_client.models.pricing_unit_get import PricingUnitGet as PricingUnitGet +from osparc_client.models.profile import Profile as Profile +from osparc_client.models.profile_update import ProfileUpdate as ProfileUpdate +from osparc_client.models.running_state import RunningState as _RunningState +from osparc_client.models.service_pricing_plan_get import ( + ServicePricingPlanGet as ServicePricingPlanGet, +) +from osparc_client.models.solver import Solver as Solver +from osparc_client.models.solver_port import SolverPort as SolverPort +from osparc_client.models.study import Study as Study +from osparc_client.models.study_port import StudyPort as StudyPort +from osparc_client.models.upload_links import UploadLinks as UploadLinks +from osparc_client.models.uploaded_part import UploadedPart as UploadedPart +from osparc_client.models.user_role_enum import UserRoleEnum as UserRoleEnum +from osparc_client.models.users_group import UsersGroup as UsersGroup +from osparc_client.models.validation_error import ValidationError as ValidationError +from osparc_client.models.wallet_get_with_available_credits import ( + WalletGetWithAvailableCredits as WalletGetWithAvailableCredits, +) +from osparc_client.models.wallet_status import WalletStatus as WalletStatus + + +# renames +TaskStates = _RunningState diff --git a/clients/python/test/e2e/conftest.py b/clients/python/test/e2e/conftest.py index 5270ea82..738146f8 100644 --- a/clients/python/test/e2e/conftest.py +++ b/clients/python/test/e2e/conftest.py @@ -19,7 +19,7 @@ from pydantic import ByteSize try: - from osparc._settings import ConfigurationModel + from osparc._settings import ConfigurationEnvVars except ImportError: pass @@ -114,7 +114,7 @@ def api_client() -> Iterable[osparc.ApiClient]: @pytest.fixture def async_client() -> Iterable[AsyncClient]: if Version(osparc.__version__) >= Version("8.0.0"): - configuration = ConfigurationModel() + configuration = ConfigurationEnvVars() host = configuration.OSPARC_API_HOST.rstrip("/") username = configuration.OSPARC_API_KEY password = configuration.OSPARC_API_SECRET diff --git a/docs/doc_entrypoint.md b/docs/doc_entrypoint.md index cee13777..65c4835f 100644 --- a/docs/doc_entrypoint.md +++ b/docs/doc_entrypoint.md @@ -67,7 +67,7 @@ with ApiClient() as api_client: # 'role': 'USER'} ``` -### Additional Resources +## Additional Resources For more in-depth usage, refer to the following resources: diff --git a/requirements.txt b/requirements.txt index 857da49f..755b2454 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +black pre-commit semver typer diff --git a/scripts/common.Makefile b/scripts/common.Makefile index 5d78058b..7a89fdfa 100644 --- a/scripts/common.Makefile +++ b/scripts/common.Makefile @@ -13,6 +13,7 @@ NOW_TIMESTAMP := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") APP_NAME := $(notdir $(CURDIR)) # Specify which openapi generator should be used to generate the clients in this repo +# Build from modified fork https://github.com/ITISFoundation/openapi-generator/tree/openapi-generator-v4.2.3 OPENAPI_GENERATOR_NAME := itisfoundation/openapi-generator-cli-openapi-generator-v4.2.3 OPENAPI_GENERATOR_TAG := v0 OPENAPI_GENERATOR_IMAGE := $(OPENAPI_GENERATOR_NAME):$(OPENAPI_GENERATOR_TAG) diff --git a/scripts/generate_version.bash b/scripts/compute_version.bash old mode 100644 new mode 100755 similarity index 97% rename from scripts/generate_version.bash rename to scripts/compute_version.bash index 3f0e982a..e45ddc42 --- a/scripts/generate_version.bash +++ b/scripts/compute_version.bash @@ -23,7 +23,7 @@ n_commits_to_merge_base=$(git rev-list --count "${merge_base}".."${current_commi # NOTE: # - we develop using post-release versioning # - i.e. 1.2.3.post0.devN where N is the number of commits with respect to last release 1.2.3) -# - Another approach would be using a pre-release version but we do not want to decide on that version +# - Another approach would be using a pre-release version but we do not want to decide on what version to release # - the releases are of the type 1.2.3 # - we never do post releases as 1.2.3.postX but instead use patches i.e. 1.2.4 # - the releases are defined using git tags (that is the case with n_commits_to_merge_base=0 )