🔨 Changes developmental releases strategy (#180)

Author: pcrespov

.gitignore

@@ -89,3 +89,8 @@ tmp*
 *ignore*
 !.dockerignore
 !.gitignore
+
+
+# api diffs
+api/openapi-*-diff.json
+api/openapi-dev.json

api/Makefile

@@ -0,0 +1,20 @@
+include ../scripts/common.Makefile
+
+
+.PHONY: openapi-diff-dev.json
+openapi-dev-diff.json: ## Diffs against newer openapi-dev.json and checks backwards compatibility
+	$(SCRIPTS_DIR)/openapi-diff.bash \
+	/specs/openapi.json \
+	/specs/openapi-dev.json \
+	--fail-on-incompatible \
+	--json=/specs/$@
+
+
+
+.PHONY: openapi-diff-dev.json
+openapi-deploy-diff.json: ## Diffs against newer openapi-dev.json and checks backwards compatibility
+	$(SCRIPTS_DIR)/openapi-diff.bash \
+	https://api.osparc.io/api/v0/openapi.json \
+	/specs/openapi.json \
+	--fail-on-incompatible \
+	--json=/specs/$@

clients/python/src/osparc/_files_api.py

@@ -29,13 +29,16 @@
     PaginationGenerator,
     compute_sha256,
     file_chunk_generator,
+    dev_features_enabled,
 )
 
 _logger = logging.getLogger(__name__)
 
 
 class FilesApi(_FilesApi):
-    """Class for interacting with files"""
+    _dev_features = [
+        "get_jobs_page",
+    ]
 
     def __init__(self, api_client: ApiClient):
         """Construct object
@@ -52,6 +55,11 @@ def __init__(self, api_client: ApiClient):
             else None
         )
 
+    def __getattr__(self, name: str) -> Any:
+        if (name in FilesApi._dev_features) and (not dev_features_enabled()):
+            raise NotImplementedError(f"FilesApi.{name} is still under development")
+        return super().__getattribute__(name)
+
     def download_file(
         self, file_id: str, *, destination_folder: Optional[Path] = None, **kwargs
     ) -> str:

clients/python/src/osparc/_solvers_api.py

@@ -14,6 +14,8 @@
     dev_features_enabled,
 )
 
+import warnings
+
 
 class SolversApi(_SolversApi):
     """Class for interacting with solvers"""
@@ -22,11 +24,6 @@ class SolversApi(_SolversApi):
         "get_jobs_page",
     ]
 
-    def __getattr__(self, name: str) -> Any:
-        if (name in SolversApi._dev_features) and (not dev_features_enabled()):
-            raise NotImplementedError(f"SolversApi.{name} is still under development")
-        return super().__getattribute__(name)
-
     def __init__(self, api_client: ApiClient):
         """Construct object
 
@@ -42,6 +39,11 @@ def __init__(self, api_client: ApiClient):
             else None
         )
 
+    def __getattr__(self, name: str) -> Any:
+        if (name in SolversApi._dev_features) and (not dev_features_enabled()):
+            raise NotImplementedError(f"SolversApi.{name} is still under development")
+        return super().__getattribute__(name)
+
     def list_solver_ports(
         self, solver_key: str, version: str, **kwargs
     ) -> List[SolverPort]:
@@ -51,7 +53,7 @@ def list_solver_ports(
         return page.items if page.items else []
 
     @dev_feature
-    def jobs(self, solver_key: str, version: str) -> PaginationGenerator:
+    def iter_jobs(self, solver_key: str, version: str, **kwargs) -> PaginationGenerator:
         """Returns an iterator through which one can iterate over
         all Jobs submitted to the solver
 
@@ -73,6 +75,7 @@ def _pagination_method():
                 version=version,
                 limit=_DEFAULT_PAGINATION_LIMIT,
                 offset=_DEFAULT_PAGINATION_OFFSET,
+                **kwargs,
             )
 
         return PaginationGenerator(
@@ -82,6 +85,16 @@ def _pagination_method():
             auth=self._auth,
         )
 
+    @dev_feature
+    def jobs(self, solver_key: str, version: str, **kwargs) -> PaginationGenerator:
+        warnings.warn(
+            "The 'jobs' method is deprecated and will be removed in a future version. "
+            "Please use 'iter_jobs' instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self.iter_jobs(solver_key, version, **kwargs)
+
     def create_job(
         self, solver_key: str, version: str, job_inputs: JobInputs, **kwargs
     ):

clients/python/src/osparc/_studies_api.py

@@ -18,6 +18,7 @@
     PaginationGenerator,
     dev_features_enabled,
 )
+import warnings
 
 _logger = logging.getLogger(__name__)
 
@@ -68,7 +69,7 @@ def clone_study(self, study_id: str, **kwargs):
         kwargs = {**kwargs, **ParentProjectInfo().model_dump(exclude_none=True)}
         return super().clone_study(study_id, **kwargs)
 
-    def studies(self, **kwargs) -> PaginationGenerator:
+    def iter_studies(self, **kwargs) -> PaginationGenerator:
         def _pagination_method():
             page_study = self.list_studies(
                 limit=_DEFAULT_PAGINATION_LIMIT,
@@ -85,6 +86,15 @@ def _pagination_method():
             auth=self._auth,
         )
 
+    def studies(self, **kwargs) -> PaginationGenerator:
+        warnings.warn(
+            "The 'studies' method is deprecated and will be removed in a future version. "
+            "Please use 'iter_studies' instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self.iter_studies(**kwargs)
+
     def get_study_job_output_logfiles(self, study_id: str, job_id: str) -> Path:
         return asyncio.run(
             self.get_study_job_output_logfiles_async(study_id=study_id, job_id=job_id)

clients/python/test/e2e/ci/e2e/e2e/postprocess.py

@@ -239,7 +239,7 @@ def clean_up_jobs(artifacts_dir: Path, retry_minutes: Optional[PositiveInt] = No
                             assert isinstance(solver, osparc.Solver)
                             assert (id_ := solver.id) is not None
                             assert (version := solver.version) is not None
-                            for job in solvers_api.jobs(id_, version):
+                            for job in solvers_api.iter_jobs(id_, version):
                                 assert isinstance(job, osparc.Job)
                                 solvers_api.delete_job(id_, version, job.id)
         except RetryError as exc:

clients/python/test/e2e/conftest.py

@@ -164,7 +164,7 @@ def sleeper_study_id(api_client: osparc.ApiClient) -> UUID:
     as input a single file containing a single integer"""
     _test_study_title = "sleeper_test_study"
     study_api = osparc.StudiesApi(api_client=api_client)
-    for study in study_api.studies():
+    for study in study_api.iter_studies():
         if study.title == _test_study_title:
             return UUID(study.uid)
     pytest.fail(f"Could not find {_test_study_title} study")

clients/python/test/e2e/test_solvers_api.py

@@ -24,7 +24,7 @@ def test_jobs(api_client: osparc.ApiClient, sleeper: osparc.Solver):
     n_jobs: int = 3
     solvers_api = osparc.SolversApi(api_client=api_client)
     # initial iterator
-    init_iter = solvers_api.jobs(sleeper.id, sleeper.version)
+    init_iter = solvers_api.iter_jobs(sleeper.id, sleeper.version)
     n_init_iter: int = len(init_iter)
     assert n_init_iter >= 0
 
@@ -36,10 +36,10 @@ def test_jobs(api_client: osparc.ApiClient, sleeper: osparc.Solver):
         )
         created_job_ids.append(job.id)
 
-    tmp_iter = solvers_api.jobs(sleeper.id, sleeper.version)
-    solvers_api.jobs(sleeper.id, sleeper.version)
+    tmp_iter = solvers_api.iter_jobs(sleeper.id, sleeper.version)
+    solvers_api.iter_jobs(sleeper.id, sleeper.version)
 
-    final_iter = solvers_api.jobs(sleeper.id, sleeper.version)
+    final_iter = solvers_api.iter_jobs(sleeper.id, sleeper.version)
     assert len(final_iter) > 0, "No jobs were available"
     assert n_init_iter + n_jobs == len(
         final_iter

scripts/generate_version.bash

@@ -10,13 +10,30 @@ set -o pipefail # don't hide errors within pipes
 trap "{ echo 'Could not generate a version file. Probably you need to sync your fork.' >&2; }" ERR
 
 release_info=$(git ls-remote --tags --refs --sort=version:refname https://github.com/ITISFoundation/osparc-simcore-clients | tail -1)
-release_version=$(echo "${release_info}" | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+')
-release_commit=$(echo "${release_info}" | grep -oE '^[[:alnum:]]+')
+released_version=$(echo "${release_info}" | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+')
+released_commit=$(echo "${release_info}" | grep -oE '^[[:alnum:]]+')
 current_commit=$(git rev-parse HEAD)
 
-#
-# Determines how many commits since the last release and adds that as `post` index.
-# WARNING: we should change Version Scheme to https://packaging.python.org/en/latest/specifications/version-specifiers/#version-scheme
-merge_base=$(git merge-base "${release_commit}" "${current_commit}")
+
+# Determines how many commits since the last release and adds that as `dev` index.
+merge_base=$(git merge-base "${released_commit}" "${current_commit}")
 n_commits_to_merge_base=$(git rev-list --count "${merge_base}".."${current_commit}")
-echo -n "${release_version}.post${n_commits_to_merge_base}"
+
+
+# NOTE:
+#   - we develop using post-release versioning
+#       - i.e. 1.2.3.post3.devN where N is the number of commits with respect to last release 1.2.3)
+#       - Another approach would be using a pre-release version but we do not want to decide on that version
+#   - the releases are of the type  1.2.3
+#   - we never do post releases as 1.2.3.postX but instead use patches i.e. 1.2.4
+#   - the releases are defined using git tags (that is the case with n_commits_to_merge_base=0 )
+#   - SEE .github/workflows/publish-python-client.yml for more details
+#
+if [ "$n_commits_to_merge_base" -gt 0 ]; then
+    echo "${released_version}.post3.dev${n_commits_to_merge_base}"
+elif [ "$n_commits_to_merge_base" -eq 0 ]; then
+    echo "${released_version}"
+else
+    echo "Error: n_commits_to_merge_base is negative. This should not happen." >&2
+    exit 1
+fi

scripts/openapi-diff.bash

@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+#
+# - https://github.com/OpenAPITools/openapi-diff
+#
+# NOTE: do not forget that the target /specs
+
+exec docker run \
+  --interactive \
+  --rm \
+  --volume="/etc/group:/etc/group:ro" \
+  --volume="/etc/passwd:/etc/passwd:ro" \
+  --user="$(id --user "$USER")":"$(id --group "$USER")" \
+  --volume "$(pwd):/specs" \
+  openapitools/openapi-diff:latest \
+  "$@"