Wkcuber with typer (#903)

* First minimal version of wkcuber with typer.

* Add executor options to all subtasks that support executor.

* Add minimal testcases for wkcuber.

* Add testcases to test_cli.py.

* Add parser for voxel size, mag and bounding box.

* Update wkcuber/README.md

Co-authored-by: Tom Herold <[email protected]>

* Update wkcuber/README.md

Co-authored-by: Tom Herold <[email protected]>

* Implement requested changes.

* Fix minor naming issues.

* Update readme.md.

* Refactor check_equality.py

* Correct wrongly imported Annotated.

* Implement requested changes and keep some of the old wkcuber modules.

* Move wkcuber into webknossos package and update tests and docs.

* Update dependencies and adapt ci workflow.

* Update local_wk_setup and remove unused methods from utils.

* Update dependencies.

* Disable typecheck for nifti image file.

* Update download functionality and fix some typing issues.

* Type further variables and adapt the parser methods.

* Create redirection package and mark wkcuber as outdated.

* Update testcases of webknossos cli tool.

* Implement further requested changes.

* Change typing of parse_path.

* disable fixme for pylint.

* Adapt description of enum in cli utils.

* Adapt docker test in ci pipeline.

* Update Dockerfile

* Update README.md

* Update webknossos/Changelog.md

Co-authored-by: Norman Rzepka <[email protected]>

* Change exit code for failing equality check.

---------

Co-authored-by: Tom Herold <[email protected]>
Co-authored-by: Norman Rzepka <[email protected]>

Author: 3 people

.dockerignore

@@ -1,4 +1,2 @@
-wkcuber/testdata
-wkcuber/testoutput
 webknossos/testdata
 webknossos/testoutput

.github/ISSUE_TEMPLATE/bug_report.md

@@ -8,7 +8,7 @@ assignees: ''
 ---
 
 ## Context
-- Affected library: e.g. webknossos or wkcuber
+- Affected library: e.g. webknossos
 <!--- In what context did your bug occur? -->
 <!--- Providing context helps us come up with a solution that is most useful. -->
 
@@ -32,4 +32,4 @@ assignees: ''
 <!--- In what environment did the bug occur? -->
 - Operating System and version: e.g. Windows 10
 - Version of webKnossos-libs (Release or Commit):
-  (can be found e.g. with `pip show webknossos` or `wkcuber --version`)
+  (can be found e.g. with `pip show webknossos --version`)

.github/ISSUE_TEMPLATE/feature_suggestion.md

@@ -8,7 +8,7 @@ assignees: ''
 ---
 
 ## Detailed Description
-- Affected library: e.g. webknossos or wkcuber
+- Affected library: e.g. webknossos
 <!--- Provide a detailed description of the change or new feature you would like to have. -->
 
 ## Use Cases & Context

.github/workflows/ci.yml

@@ -17,7 +17,6 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       webknossos: ${{ steps.filter.outputs.webknossos || github.ref == 'refs/heads/master' }}
-      wkcuber: ${{ steps.filter.outputs.wkcuber || github.ref == 'refs/heads/master' }}
       cluster_tools: ${{ steps.filter.outputs.cluster_tools || github.ref == 'refs/heads/master' }}
     steps:
     - uses: actions/checkout@v3
@@ -27,8 +26,6 @@ jobs:
         filters: |
           webknossos:
             - 'webknossos/**'
-          wkcuber:
-            - 'wkcuber/**'
           cluster_tools:
             - 'cluster_tools/**'
 
@@ -180,118 +177,10 @@ jobs:
         git diff --no-ext-diff --quiet --exit-code
         [[ -z $(git status -s) ]]
 
-  wkcuber_linux:
-    needs: [changes]
-    if: |
-      ${{ needs.changes.outputs.cluster_tools == 'true' }} ||
-      ${{ needs.changes.outputs.webknossos == 'true' }} ||
-      ${{ needs.changes.outputs.wkcuber == 'true' }}
-    runs-on: ubuntu-latest
-    strategy:
-      max-parallel: 4
-      matrix:
-        python-version: ["3.8", "3.9", "3.10"]
-    defaults:
-      run:
-        working-directory: wkcuber
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Install dependencies
-      run: |
-        pip install -r ../requirements.txt
-        poetry install
-
-    - name: Check formatting
-      run: ./format.sh check
-      if: ${{ needs.changes.outputs.wkcuber == 'true' }}
-
-    - name: Lint code
-      run: ./lint.sh
-      if: ${{ needs.changes.outputs.wkcuber == 'true' }}
-
-    - name: Check typing
-      run: ./typecheck.sh
-
-    - name: Run tests
-      run: ./test.sh
-
-    - name: Check if git is dirty
-      run: |
-        git diff --no-ext-diff --quiet --exit-code
-        [[ -z $(git status -s) ]]
-
-  wkcuber_win:
-    # Caution! The Windows VM seems to be running out of storage rather quickly.
-    runs-on: windows-latest
-    needs: [changes]
-    if: |
-      ${{ needs.changes.outputs.cluster_tools == 'true' }} ||
-      ${{ needs.changes.outputs.webknossos == 'true' }} ||
-      ${{ needs.changes.outputs.wkcuber == 'true' }}
-    strategy:
-      max-parallel: 4
-      matrix:
-        python-version: ["3.8", "3.9", "3.10"]
-    defaults:
-      run:
-        working-directory: wkcuber
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Install dependencies
-      shell: bash
-      run: |
-        pip install -r ../requirements.txt
-        poetry install
-
-    - name: Run tests
-      continue-on-error: true
-      shell: bash
-      run: poetry run "C:\Program Files\Git\bin\bash.EXE" --noprofile --norc ./test.sh
-
-  wkcuber_mac:
-    runs-on: macos-latest
-    needs: [changes]
-    if: |
-      ${{ needs.changes.outputs.cluster_tools == 'true' }} ||
-      ${{ needs.changes.outputs.webknossos == 'true' }} ||
-      ${{ needs.changes.outputs.wkcuber == 'true' }}
-    strategy:
-      max-parallel: 4
-      matrix:
-        python-version: ["3.8", "3.9", "3.10"]
-    defaults:
-      run:
-        working-directory: wkcuber
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Install dependencies
-      run: |
-        pip install -r ../requirements.txt
-        poetry install
 
-    - name: Run tests
-      run: ./test.sh
 
-  wkcuber_docker:
-    needs: [cluster_tools, webknossos_linux, wkcuber_linux, wkcuber_win, wkcuber_mac]
+  webknossos_cli_docker:
+    needs: [cluster_tools, webknossos_linux]
     if: |
       always() &&
       !contains(needs.*.result, 'failure') &&
@@ -309,29 +198,25 @@ jobs:
     - name: Write version file
       run: |
         pip install -r requirements.txt
-        pushd wkcuber
+        pushd webknossos
 
         PKG_VERSION="$(dunamai from git)"
-        echo "__version__ = '$PKG_VERSION'" > ./wkcuber/version.py
+        echo "__version__ = '$PKG_VERSION'" > ./webknossos/version.py
         poetry version "$PKG_VERSION"
 
         popd
 
     - name: Build docker image
-      run: docker build -t scalableminds/webknossos-cuber:$GITHUB_SHA -f wkcuber/Dockerfile .
+      run: docker build -t scalableminds/webknossos-cli:$GITHUB_SHA -f webknossos/Dockerfile .
 
     - name: Smoke test docker
       run: |
         docker run --rm \
-          -v$(pwd)/wkcuber/testdata:/app/testdata \
           -v$(pwd)/webknossos/testdata:/webknossos/testdata \
-          scalableminds/webknossos-cuber:$GITHUB_SHA \
-          wkcuber.cubing \
+          scalableminds/webknossos-cli:$GITHUB_SHA \
+          webknossos convert \
             --jobs 2 \
-            --batch_size 8 \
-            --layer_name color \
-            --chunks_per_shard 32 \
-            --voxel_size 1,1,1 \
+            --voxel-size 1,1,1 \
             testdata/tiff testoutput/tiff
 
     - name: Login to docker
@@ -343,33 +228,33 @@ jobs:
 
     - name: Push docker images
       run: |
-        docker push scalableminds/webknossos-cuber:$GITHUB_SHA
+        docker push scalableminds/webknossos-cli:$GITHUB_SHA
 
     - name: Push docker images (for tag)
       if: github.ref_type == 'tag'
       run: |
         CI_TAG=$(git describe --tags)
         docker tag \
-          scalableminds/webknossos-cuber:$GITHUB_SHA \
-          scalableminds/webknossos-cuber:$CI_TAG
-        docker push scalableminds/webknossos-cuber:$CI_TAG
+          scalableminds/webknossos-cli:$GITHUB_SHA \
+          scalableminds/webknossos-cli:$CI_TAG
+        docker push scalableminds/webknossos-cli:$CI_TAG
         docker tag \
-          scalableminds/webknossos-cuber:$GITHUB_SHA \
-          scalableminds/webknossos-cuber:latest
-        docker push scalableminds/webknossos-cuber:latest
+          scalableminds/webknossos-cli:$GITHUB_SHA \
+          scalableminds/webknossos-cli:latest
+        docker push scalableminds/webknossos-cli:latest
 
     - name: Push docker images (for branch)
       if: github.ref_type == 'branch'
       run: |
         CI_BRANCH=${GITHUB_HEAD_REF:-$GITHUB_REF_NAME}
         NORMALIZED_CI_BRANCH=${CI_BRANCH//[\/-]/_}
         docker tag \
-          scalableminds/webknossos-cuber:$GITHUB_SHA \
-          scalableminds/webknossos-cuber:$NORMALIZED_CI_BRANCH
-        docker push scalableminds/webknossos-cuber:$NORMALIZED_CI_BRANCH
+          scalableminds/webknossos-cli:$GITHUB_SHA \
+          scalableminds/webknossos-cli:$NORMALIZED_CI_BRANCH
+        docker push scalableminds/webknossos-cli:$NORMALIZED_CI_BRANCH
 
   docs:
-    needs: [cluster_tools, webknossos_linux, wkcuber_linux, wkcuber_win, wkcuber_mac]
+    needs: [cluster_tools, webknossos_linux]
     runs-on: ubuntu-latest
     if: |
       always() &&
@@ -430,7 +315,7 @@ jobs:
 
 
   pypi_and_gh_release:
-    needs: [cluster_tools, webknossos_linux, wkcuber_linux, wkcuber_win, wkcuber_mac]
+    needs: [cluster_tools, webknossos_linux]
     if: |
       always() &&
       !contains(needs.*.result, 'failure') &&
@@ -470,7 +355,7 @@ jobs:
           prerelease: false
 
   complete:
-    needs: [cluster_tools, webknossos_linux, wkcuber_linux, wkcuber_win, wkcuber_mac, wkcuber_docker, docs, pypi_and_gh_release]
+    needs: [cluster_tools, webknossos_linux, webknossos_cli_docker, docs, pypi_and_gh_release]
     if: always()
     runs-on: ubuntu-latest
     steps:

CONTRIBUTING.md

@@ -98,7 +98,6 @@ The [WEBKNOSSOS-libs repository](https://github.com/scalableminds/webknossos-lib
 
 * `cluster_tools`
 * `webknossos`
-* `wkcuber`
 * (`docs`, see below for **Documentation**)
 
 See below for specifics of the different packages. Let's have a look at the common tooling first:
@@ -157,13 +156,6 @@ To re-generate the code, run
 ./generate_client.sh
 ```
 
-
-#### `wkcuber` package
-
-Simply use the default scripts mentioned above, such as
-`./format.sh`, `./lint.sh`, `./typecheck.sh`, `./test.sh`.
-
-
 #### `cluster_tools` package
 
 For testing the `slurm` setup a docker-compose setup is available. Please see the [respective Readme](https://github.com/scalableminds/webknossos-libs/blob/master/cluster_tools/README.md) for details.
@@ -177,7 +169,7 @@ We render a common documentation for WEBKNOSSOS itself and WEBKNOSSOS-libs from
 
 * `docs/src/webknossos`: Server & Website documentation, linked from the [webknossos repository](https://github.com/scalableminds/webknossos) (must be available under `docs/wk-repo`, see below).
 * `docs/src/api`: Generated using [pdoc](https://pdoc.dev) from Python docstrings.
-* `docs/src/webknossos-py` & `docs/src/wkcuber`: Documentation for the respective Python Packages
+* `docs/src/webknossos-py`: Documentation for the Python Package
 
 The structure of the documentation page is given by `docs/mkdocs.yml`.
 

Makefile

@@ -1,6 +1,6 @@
-packages_by_priority := webknossos wkcuber cluster_tools docs
-packages_by_dependency := cluster_tools webknossos wkcuber docs
-code_packages := cluster_tools webknossos wkcuber
+packages_by_priority := webknossos cluster_tools docs
+packages_by_dependency := cluster_tools webknossos docs
+code_packages := cluster_tools webknossos
 
 define in_each_pkg_by_dependency
   for PKG in $(packages_by_dependency); do echo $$PKG; cd $$PKG; $1; cd ..; done

README.md

@@ -8,26 +8,29 @@
 <img align="right" src="https://static.webknossos.org/logos/webknossos-icon-only.svg" alt="WEBKNOSSOS Logo" width="100" height="100"/>
 
 ## [WEBKNOSSOS Python API](webknossos)
+
+### API
+
 Python API for working with [WEBKNOSSOS](https://webknossos.org) datasets, annotations, and for WEBKNOSSOS server interactions.
 
 Use this for:
 - reading/writing/manipulating raw 2D/3D image data and volume annotations/segmentation in WEBKNOSSOS wrap (*.wkw) format
 - handling/manipulation of WEBKNOSSOS datasets
 - reading/writing/manipulating WEBKNOSSOS skeleton annotations (*.nml)
 - up- & downloading annotations and datasets from your WEBKNOSSOS instance
-- executing any of the wkcuber operations from below from your code
 
 [Read more in the docs.](https://docs.webknossos.org/webknossos-py/)
 
-## [WEBKNOSSOS cuber (wkcuber) CLI](wkcuber)
+### CLI
+
 CLI tool for creating and manipulating [WEBKNOSSOS](https://webknossos.org) [WKW](https://github.com/scalableminds/webknossos-wrap) datasets. WKW is a container format for efficiently storing large-scale 3D images as found in microscopy data.
 
 Use this for:
 - converting Tiff-stacks and other data formats for volume image data to WEBKNOSSOS-compatible *.wkw files from the CLI
 - up/downsampling of *.wkw files to different magnification levels (image pyramid) from the CLI
 - compressing your *.wkw files to save disk space from the CLI
 
-[Read more in the docs.](https://docs.webknossos.org/wkcuber)
+[Read more in the docs.](https://docs.webknossos.org/webknossos-py/)
 
 ## [Cluster Tools](cluster_tools)
 The `cluster_tools` package provides python `Executor` classes for distributing tasks on a slurm cluster or via multi processing.

docs/generate.sh

@@ -17,25 +17,19 @@ fi
 
 export PDOC_CLASS_MODULES="$(poetry run python get_keyword_mapping.py)"
 if [ $# -eq 1 ] && [ "$1" = "--api" ]; then
-    poetry run pdoc ../webknossos/webknossos ../wkcuber/wkcuber -t pdoc_templates/pure_pdoc -h 0.0.0.0 -p 8196
+    poetry run pdoc ../webknossos/webknossos -t pdoc_templates/pure_pdoc -h 0.0.0.0 -p 8196
 else
     rm -rf src/api
-    poetry run pdoc ../webknossos/webknossos ../wkcuber/wkcuber -t pdoc_templates/with_mkdocs -o src/api
+    poetry run pdoc ../webknossos/webknossos -t pdoc_templates/with_mkdocs -o src/api
     # rename .html files to .md
     find src/api -iname "*.html" -exec sh -c 'mv "$0" "${0%.html}.md"' {} \;
     # assert that API docs are written
     webknossos_files="$(find src/api/webknossos -type f -name "*.md" | wc -l)"
-    wkcuber_files="$(find src/api/wkcuber -type f -name "*.md" | wc -l)"
     if ! [ "$webknossos_files" -gt "50" ]; then
        echo "Error: There are too few ($webknossos_files, expected > 80) files in src/api/webknossos,"
        echo "probably there was an error with pdoc before!"
        exit 1
     fi
-    if ! [ "$wkcuber_files" -gt "25" ]; then
-       echo "There are too few ($wkcuber_files, expected > 25) files in src/api/wkcuber,"
-       echo "probably there was an error with pdoc before!"
-       exit 1
-    fi
     if [ $# -eq 1 ] && [ "$1" = "--persist" ]; then
         PYTHONPATH=$PYTHONPATH:. poetry run mkdocs build
     else

docs/mkdocs.yml

@@ -120,6 +120,8 @@ nav:
               - User: api/webknossos/administration/user.md
               - Project: api/webknossos/administration/project.md
               - Task: api/webknossos/administration/task.md
+      - CLI Reference:
+        - Overview: cli/index.md
       - Version Changelog:
           - webknossos-py/stability_policy.md
           - webknossos-py/changelog.md
@@ -133,14 +135,6 @@ nav:
               - Commercial Support: https://scalableminds.com" target="_blank
           - GitHub: https://github.com/scalableminds/webknossos-libs/tree/master/webknossos" target="_blank
           - Blog: https://medium.com/scalableminds" target="_blank
-  - WEBKNOSSOS Cuber:
-      - wkcuber CLI: wkcuber/index.md
-      - wkcuber/changelog.md
-      - Getting Help:
-          - Community Support: https://forum.image.sc/tag/webknossos" target="_blank
-          - Email Support: mailto:[email protected]
-          - Commercial Support: https://scalableminds.com" target="_blank
-      - GitHub: https://github.com/scalableminds/webknossos-libs/tree/master/wkcuber" target="_blank
 
 plugins:
   - search