webKnossos Python Libary: initial setup (#398)

* default to -1 for largestSegmentId

* Update Changelog.md

* minor fix

* formatting

* setup folder structure and basic CI checks

* use dunamai instead of setuptools_scm

* fix? gh actions

* trigger CI

* print error

* fix properties path after merge

* setup folder structure, copy dataset api

* fix linting

* rename wklib to webknossos

* formatting

* remove duplicated tests

* set license

* use single CI workflow, refactor publish for all projects

* webknossos CI only on changes, fix docker build

* fix docker smoke test

* fix changed check in CI

* checkout HEAD~1 in CI

* fix publish.sh

* use webknossos import in wkcuber

* pylint: ignore wildcard imports

* use public links in examples

* type fixes

* formatting

* better CI handling for unchanged files

* update changelog

* fix client readme

Author: jstriebel

.dockerignore

@@ -1,2 +1,4 @@
-testdata
-testoutput
+wkcuber/testdata
+wkcuber/testoutput
+webknossos/testdata
+webknossos/testoutput

.github/dependabot.yml

@@ -1,7 +1,7 @@
 version: 2
 updates:
 - package-ecosystem: pip
-  directory: "/"
+  directory: "/wkcuber"
   schedule:
     interval: daily
   open-pull-requests-limit: 10

.github/workflows/main.yml renamed to .github/workflows/ci.yml

@@ -1,14 +1,77 @@
 name: CI
 
-on: [push]
+on: push
 
 jobs:
-  build:
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      webknossos: ${{ steps.filter.outputs.webknossos }}
+      wkcuber: ${{ steps.filter.outputs.wkcuber }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: dorny/paths-filter@v2
+      id: filter
+      with:
+        filters: |
+          webknossos:
+            - 'webknossos/**'
+          wkcuber:
+            - 'wkcuber/**'
+
+  webknossos_linux:
+    needs: changes
+    if: ${{ needs.changes.outputs.webknossos == 'true' }}
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: [3.7, 3.8]
+    defaults:
+      run:
+        working-directory: webknossos
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+        architecture: 'x64'
+
+    - name: Install dependencies
+      run: |
+        pip install poetry
+        poetry install
+
+    - name: Check formatting
+      run: poetry run ./format.sh check
+
+    - name: Lint code
+      run: poetry run pylint -j4 webknossos
+
+    - name: Check typing
+      run: |
+        poetry run ./typecheck.sh
+
+    - name: Python tests
+      run: poetry run pytest tests
+
+    - name: Check if git is dirty
+      run: |
+        git diff --no-ext-diff --quiet --exit-code
+        [[ -z $(git status -s) ]]
+
+  wkcuber_linux:
+    needs: [changes]
     runs-on: ubuntu-latest
     strategy:
       max-parallel: 4
       matrix:
         python-version: [3.7, 3.8]
+    defaults:
+      run:
+        working-directory: wkcuber
 
     steps:
     - uses: actions/checkout@v2
@@ -27,9 +90,11 @@ jobs:
 
     - name: Check formatting
       run: poetry run black --check .
+      if: ${{ needs.changes.outputs.wkcuber == 'true' }}
 
     - name: Lint code
       run: poetry run pylint -j4 wkcuber
+      if: ${{ needs.changes.outputs.wkcuber == 'true' }}
 
     - name: Check typing
       run: |
@@ -45,7 +110,7 @@ jobs:
       if: matrix.python-version == '3.8'
       run: |
         docs/api.sh --persist
-    
+
     - name: Push docs (for branch)
       if: startsWith(github.event.ref, 'refs/heads') && matrix.python-version == '3.8'
       env:
@@ -72,20 +137,16 @@ jobs:
         git diff --no-ext-diff --quiet --exit-code
         [[ -z $(git status -s) ]]
 
-    - name: Publish python package
-      if: startsWith(github.event.ref, 'refs/tags') && matrix.python-version == '3.8'
-      env:
-        PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      run: ./publish.sh
-
-  build_win:
+  wkcuber_win:
     # Caution! The Windows VM seems to be running out of storage rather quickly.
     runs-on: windows-latest
     strategy:
       max-parallel: 4
       matrix:
         python-version: [3.7, 3.8]
+    defaults:
+      run:
+        working-directory: wkcuber
 
     steps:
     - uses: actions/checkout@v2
@@ -112,12 +173,15 @@ jobs:
       shell: bash
       run: tests/scripts/all_tests.sh
 
-  build_mac:
+  wkcuber_mac:
     runs-on: macos-latest
     strategy:
       max-parallel: 4
       matrix:
         python-version: [3.7, 3.8]
+    defaults:
+      run:
+        working-directory: wkcuber
 
     steps:
     - uses: actions/checkout@v2
@@ -140,19 +204,19 @@ jobs:
     - name: CLI tests
       run: tests/scripts/all_tests.sh
 
-  docker:
-    needs: [build, build_win, build_mac]
+  wkcuber_docker:
+    needs: [webknossos_linux, wkcuber_linux, wkcuber_win, wkcuber_mac]
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v2
     - name: Build docker image
-      run: docker build -t scalableminds/webknossos-cuber:$GITHUB_SHA .
+      run: docker build -t scalableminds/webknossos-cuber:$GITHUB_SHA -f wkcuber/Dockerfile .
 
     - name: Smoke test docker
       run: |
         docker run --rm \
-          -v$(pwd)/testdata:/app/testdata \
+          -v$(pwd)/wkcuber/testdata:/app/testdata \
           scalableminds/webknossos-cuber:$GITHUB_SHA \
           wkcuber.cubing \
             --jobs 2 \
@@ -196,3 +260,25 @@ jobs:
             scalableminds/webknossos-cuber:latest
           docker push scalableminds/webknossos-cuber:latest
         fi
+
+  pypi:
+    needs: [webknossos_linux, wkcuber_linux, wkcuber_win, wkcuber_mac]
+    if: startsWith(github.event.ref, 'refs/tags')
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 4
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+        architecture: 'x64'
+    - name: Install dependencies
+      run: pip3 install -r requirements.txt
+    - name: Publish python packages
+      env:
+        PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
+        PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      run: ./publish.sh

Dockerfile

@@ -1,144 +1,12 @@
-# webKnossos cuber (wkcuber)
-[![PyPI version](https://img.shields.io/pypi/v/wkcuber)](https://pypi.python.org/pypi/wkcuber)
-[![Supported Python Versions](https://img.shields.io/pypi/pyversions/wkcuber.svg)](https://pypi.python.org/pypi/wkcuber)
-[![Build Status](https://img.shields.io/github/workflow/status/scalableminds/webknossos-cuber/CI/master)](https://github.com/scalableminds/webknossos-cuber/actions?query=workflow%3A%22CI%22)
-[![Code Style](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+# webKnossos-libs
+<img align="right" src="https://static.webknossos.org/images/oxalis.svg" alt="webKnossos Logo" />
 
-Python library for creating and working with [webKnossos](https://webknossos.org) [WKW](https://github.com/scalableminds/webknossos-wrap) datasets. WKW is a container format for efficiently storing large, scale 3D image data as found in (electron) microscopy.
-
-The tools are modular components to allow easy integration into existing pipelines and workflows.
-
-## Features
-
-* `wkcuber`: Convert supported input files to fully ready WKW datasets (includes type detection, downsampling, compressing and metadata generation)
-* `wkcuber.convert_image_stack_to_wkw`: Convert image stacks to fully ready WKW datasets (includes downsampling, compressing and metadata generation)
-* `wkcuber.export_wkw_as_tiff`: Convert WKW datasets to a tiff stack (writing as tiles to a `z/y/x.tiff` folder structure is also supported)
-* `wkcuber.cubing`: Convert image stacks (e.g., `tiff`, `jpg`, `png`, `dm3`, `dm4`) to WKW cubes
-* `wkcuber.tile_cubing`: Convert tiled image stacks (e.g. in `z/y/x.ext` folder structure) to WKW cubes
-* `wkcuber.convert_knossos`: Convert KNOSSOS cubes to WKW cubes
-* `wkcuber.convert_nifti`: Convert NIFTI files to WKW files (Currently without applying transformations).
-* `wkcuber.downsampling`: Create downsampled magnifications (with `median`, `mode` and linear interpolation modes). Downsampling compresses the new magnifications by default (disable via `--no-compress`).
-* `wkcuber.compress`: Compress WKW cubes for efficient file storage (especially useful for segmentation data)
-* `wkcuber.metadata`: Create (or refresh) metadata (with guessing of most parameters)
-* `wkcuber.recubing`: Read existing WKW cubes in and write them again specifying the WKW file length. Useful when dataset was written e.g. with file length 1.
-* `wkcuber.check_equality`: Compare two WKW datasets to check whether they are equal (e.g., after compressing a dataset, this task can be useful to double-check that the compressed dataset contains the same data).
-* Most modules support multiprocessing
-
-## Supported input formats
-
-* Standard image formats, e.g. `tiff`, `jpg`, `png`, `bmp`
-* Proprietary image formats, e.g. `dm3`
-* Tiled image stacks (used for Catmaid)
-* KNOSSOS cubes
-* NIFTI files
-
-## Installation
-### Python 3 with pip from PyPi
-- `wkcuber` requires at least Python 3.6+
-
-```
-# Make sure to have lz4 installed:
-# Mac: brew install lz4
-# Ubuntu/Debian: apt-get install liblz4-1
-# CentOS/RHEL: yum install lz4
-
-pip install wkcuber
-```
-
-### Docker
-Use the CI-built image: [scalableminds/webknossos-cuber](https://hub.docker.com/r/scalableminds/webknossos-cuber/). Example usage `docker run -v <host path>:/data --rm scalableminds/webknossos-cuber wkcuber --layer_name color --scale 11.24,11.24,25 --name great_dataset /data/source/color /data/target`.
-
-
-## Usage
-
-```
-# Convert arbitrary, supported input files into wkw datasets. This sets reasonable defaults, but see other commands for customization.
-python -m wkcuber \
-  --scale 11.24,11.24,25 \
-  data/source data/target
-
-# Convert image stacks into wkw datasets
-python -m wkcuber.convert_image_stack_to_wkw \
-  --layer_name color \
-  --scale 11.24,11.24,25 \
-  --name great_dataset \
-  data/source/color data/target
-
-# Convert image files to wkw cubes
-python -m wkcuber.cubing --layer_name color data/source/color data/target
-python -m wkcuber.cubing --layer_name segmentation data/source/segmentation data/target
-
-# Convert tiled image files to wkw cubes
-python -m wkcuber.tile_cubing --layer_name color data/source data/target
-
-# Convert Knossos cubes to wkw cubes
-python -m wkcuber.convert_knossos --layer_name color data/source/mag1 data/target
+Collection of libraries around webKnossos, please see the respective subfolders:
 
-# Convert NIFTI file to wkw file
-python -m wkcuber.convert_nifti --layer_name color --scale 10,10,30 data/source/nifti_file data/target
+## [webKnossos cuber (wkcuber)](wkcuber)
+[![PyPI version](https://img.shields.io/pypi/v/wkcuber)](https://pypi.python.org/pypi/wkcuber) [![Supported Python Versions](https://img.shields.io/pypi/pyversions/wkcuber.svg)](https://pypi.python.org/pypi/wkcuber)
 
-# Convert folder with NIFTI files to wkw files
-python -m wkcuber.convert_nifti --color_file one_nifti_file --segmentation_file --scale 10,10,30 another_nifti data/source/ data/target
-
-# Create downsampled magnifications
-python -m wkcuber.downsampling --layer_name color data/target
-python -m wkcuber.downsampling --layer_name segmentation --interpolation_mode mode data/target
-
-# Compress data in-place (mostly useful for segmentation)
-python -m wkcuber.compress --layer_name segmentation data/target
-
-# Compress data copy (mostly useful for segmentation)
-python -m wkcuber.compress --layer_name segmentation data/target data/target_compress
-
-# Create metadata
-python -m wkcuber.metadata --name great_dataset --scale 11.24,11.24,25 data/target
-
-# Refresh metadata so that new layers and/or magnifications are picked up
-python -m wkcuber.metadata --refresh data/target
-
-# Recubing an existing dataset
-python -m wkcuber.recubing --layer_name color --dtype uint8 /data/source/wkw /data/target
-
-# Check two datasets for equality
-python -m wkcuber.check_equality /data/source /data/target
-```
-
-### Parallelization
-
-Most tasks can be configured to be executed in a parallelized manner. Via `--distribution_strategy` you can pass `multiprocessing` or `slurm`. The first can be further configured with `--jobs` and the latter via `--job_resources='{"mem": "10M"}'`. Use `--help` to get more information.
-
-## Development
-Make sure to install all the required dependencies using Poetry:
-```
-pip install poetry
-poetry install
-```
-
-Please, format, lint, and unit test your code changes before merging them.
-```
-poetry run black .
-poetry run pylint -j4 wkcuber
-poetry run pytest tests
-```
-
-Please, run the extended test suite:
-```
-tests/scripts/all_tests.sh
-```
-
-PyPi releases are automatically pushed when creating a new Git tag/Github release. 
-
-## API documentation
-Check out the [latest version of the API documentation](https://static.webknossos.org/lib-docs/master/wkcuber/api.html).
-
-### Generate the API documentation
-Run `docs/api.sh` to open a server displaying the API docs. `docs/api.sh --persist` persists the html to `docs/api`.
-
-## Test Data Credits
-Excerpts for testing purposes have been sampled from:
-- Dow Jacobo Hossain Siletti Hudspeth (2018). **Connectomics of the zebrafish's lateral-line neuromast reveals wiring and miswiring in a simple microcircuit.** eLife. [DOI:10.7554/eLife.33988](https://elifesciences.org/articles/33988)
-- Zheng Lauritzen Perlman Robinson Nichols Milkie Torrens Price Fisher Sharifi Calle-Schuler Kmecova Ali Karsh Trautman Bogovic Hanslovsky Jefferis Kazhdan Khairy Saalfeld Fetter Bock (2018). **A Complete Electron Microscopy Volume of the Brain of Adult Drosophila melanogaster.** Cell. [DOI:10.1016/j.cell.2018.06.019](https://www.cell.com/cell/fulltext/S0092-8674(18)30787-6). License: [CC BY-NC 4.0](https://creativecommons.org/licenses/by-nc/4.0/)
+Python library for creating and working with [webKnossos](https://webknossos.org) [WKW](https://github.com/scalableminds/webknossos-wrap) datasets. WKW is a container format for efficiently storing large, scale 3D image data as found in (electron) microscopy.
 
-## License
-AGPLv3
-Copyright scalable minds
+## [webKnossos](webknossos) Python Library
+:construction: WIP