Remote Dataset (#723)

* update webknossos client, use floats for scale

* WIP

* add tests

* add changelog entry

* fix docs destination for branch builds

* same for docker

* use GITHUB_HEAD_REF with GITHUB_REF_NAME as a fallback

* apply PR feedback

* update changelog

* update examples

* update outdated cassettes

* Update webknossos/webknossos/dataset/dataset.py

Co-authored-by: Norman Rzepka <[email protected]>

* make webknossos_context reentrant

* Apply suggestions from code review

Co-authored-by: Valentin Pinkau <[email protected]>

* fix json body type of datasetUpdateTeam client method

* call _ensure_writable where needed, re-order methods for docs

* add read_only property to dataset and layer

Co-authored-by: Norman Rzepka <[email protected]>
Co-authored-by: Valentin Pinkau <[email protected]>

Author: 3 people

.github/workflows/ci.yml

@@ -326,7 +326,7 @@ jobs:
     - name: Push docker images (for branch)
       if: github.ref_type == 'branch'
       run: |
-        CI_BRANCH=${GITHUB_REF##*/}
+        CI_BRANCH=${GITHUB_HEAD_REF:-$GITHUB_REF_NAME}
         NORMALIZED_CI_BRANCH=${CI_BRANCH//[\/-]/_}
         docker tag \
           scalableminds/webknossos-cuber:$GITHUB_SHA \
@@ -375,7 +375,7 @@ jobs:
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         AWS_DEFAULT_REGION: "eu-west-1"
       run: |
-        CI_BRANCH=${GITHUB_REF##*/}
+        CI_BRANCH=${GITHUB_HEAD_REF:-$GITHUB_REF_NAME}
         NORMALIZED_CI_BRANCH=${CI_BRANCH//[\/-]/_}
         aws s3 sync --acl public-read docs/out s3://static.webknossos.org/docs/${NORMALIZED_CI_BRANCH}
 

docs/overrides/main.html

@@ -43,11 +43,11 @@
 			font-style: normal;
 		}
 		:root {
-			--md-text-font-family: "Titillium Web";
-			--md-code-font-family: "Roboto Mono";
+			--md-text-font: "Titillium Web";
+			--md-code-font: "Roboto Mono";
 		}
 		.pdoc .attr {
-			font-family: var(--md-code-font-family);
+			font-family: var(--md-code-font);
 		}
 	</style>
 {% endblock styles %}

docs/pdoc_templates/with_mkdocs/frame.html.jinja2

@@ -50,6 +50,36 @@ pdoc-toc:
 {% endif %}
 ---
 <div>
-    {% block body %}{% endblock %}
-    {% block style %}{% endblock %}
+    {% block body %}
+        {% block content %}{% endblock %}
+        {% if module and module.submodules %}
+            <h2 id="submodules">Submodules</h2>
+            <ul>
+                {% for submodule in module.submodules %}
+                    <li>{{ submodule.taken_from | link }}</li>
+                {% endfor %}
+            </ul>
+        {% endif %}
+    {% endblock %}
+    {% filter minify_css | indent %}
+        {% block style %}
+            {#
+            Customizations:
+            * removed include of resources/bootstrap-reboot.min.css
+            * removed block style_layout with include of layout.css
+            #}
+            <style>/*! syntax-highlighting.css */{% include "syntax-highlighting.css" %}</style>
+            {#
+            The style_pdoc, style_theme, style_layout, and style_content Jinja2 blocks are deprecated and will be
+            removed in a future release. Custom templates should either provide alternatives for the specific CSS files,
+            or append their own styles by providing `custom.css` (see examples/custom-template/).
+            #}
+            {% block style_pdoc %}
+                {% block style_theme %}<style>/*! theme.css */{% include "theme.css" %}</style>{% endblock %}
+                {% block style_content %}<style>/*! content.css */{% include "content.css" %}</style>{% endblock %}
+                {# Use this file in your custom template directory to add additional CSS styling: #}
+                <style>/*! custom.css */{% include "custom.css" %}</style>
+            {% endblock %}
+        {% endblock %}
+    {% endfilter %}
 </div>

docs/pdoc_templates/with_mkdocs/module.html.jinja2

@@ -4,61 +4,3 @@
 
 {% block nav %}
 {% endblock %}
-
-{% block style_layout %}{% endblock %}
-
-## The member and module_contents blocks are just to add extra divs for classes,
-## so that classes have a correct anchor when using mkdocs
-{% defaultmacro member(doc,add_div) %}
-    {% if add_div %}
-    <div id="{{ doc.qualname or doc.name }}">
-    {% endif %}
-    {% if doc.type == "class" %}
-        {{ class(doc) }}
-    {% elif doc.type == "function" %}
-        {{ function(doc) }}
-    {% elif doc.type == "module" %}
-        {{ submodule(doc) }}
-    {% else %}
-        {{ variable(doc) }}
-    {% endif %}
-    {% if doc.type != "variable" %}
-        {{ view_source(doc) }}
-    {% endif %}
-    {% if add_div %}
-    </div>
-    {% endif %}
-    {{ docstring(doc) }}
-{% enddefaultmacro %}
-
-{% block module_contents %}
-    {% for m in module.flattened_own_members if is_public(m) | trim %}
-        <section>
-            {{ member(m,true) }}
-            {% if m.type == "class" %}
-                {% for m in m.own_members if m.type != "class" and is_public(m) | trim %}
-                    <div id="{{ m.qualname }}" class="classattr">
-                        {{ member(m,false) }}
-                    </div>
-                {% endfor %}
-                {% set inherited_members = inherited(m) | trim %}
-                {% if inherited_members %}
-                    <div class="inherited">
-                        <h5>Inherited Members</h5>
-                        <dl>
-                            {{ inherited_members }}
-                        </dl>
-                    </div>
-                {% endif %}
-            {% endif %}
-        </section>
-    {% endfor %}
-    {% if module and module.submodules %}
-        <h2 id="submodules">Submodules</h2>
-        <ul>
-            {% for submodule in module.submodules %}
-                <li>{{ submodule.taken_from | link }}</li>
-            {% endfor %}
-        </ul>
-        {% endif %}
-{% endblock %}

docs/poetry.lock

@@ -10,10 +10,10 @@ python = "^3.7,>=3.7.1"
 
 [tool.poetry.dev-dependencies]
 LinkChecker = "^10.0.1"
-mkdocs = "1.2.2"
-mkdocs-material = "^7.3.0"
+mkdocs = "^1.3.0"
+mkdocs-material = "^8.2.13"
 mkdocs-redirects = "^1.0.3"
-pdoc = { git = "https://github.com/mitmproxy/pdoc.git", rev = "1574222ab0568e072dc04e3569027d39aa124256"}
+pdoc = "^11.1.0"
 webknossos = { path = "../webknossos/", develop = true }
 wkcuber = { path = "../wkcuber/", develop = true }
 

docs/pyproject.toml

@@ -13,8 +13,17 @@ For upgrade instructions, please check the respective *Breaking Changes* section
 [Commits](https://github.com/scalableminds/webknossos-libs/compare/v0.9.24...HEAD)
 
 ### Breaking Changes
+- `Dataset.upload()` now returns a `RemoteDataset` instead instead of the URL string. You can get the URL via `remote_ds.url`. [#723](https://github.com/scalableminds/webknossos-libs/pull/723)
+- `User.teams` now is a tuple instead of a list. [#723](https://github.com/scalableminds/webknossos-libs/pull/723)
+- The deprecated `download_dataset` function now requires the `organization_id` argument. [#723](https://github.com/scalableminds/webknossos-libs/pull/723)
 
 ### Added
+- Added `Dataset.open_remote()`, which returns an object of the new `RemoteDataset`. [#723](https://github.com/scalableminds/webknossos-libs/pull/723)
+  This can
+  * give the webknossos URL for the dataset as `remote_ds.url`,
+  * read image data via the webknossos zarr interface, using the inherited `Dataset` methods, and
+  * read and change the following dataset metadata as properties: `display_name`, `description`, `tags`, `is_public`, `sharing_token`, `allowed_teams`.
+- `Team` instances also contain the `organization_id`. [#723](https://github.com/scalableminds/webknossos-libs/pull/723)
 
 ### Changed