Merge pull request #37 from SpikeInterface/pykilosort2

Pykilosort2 container

Author: alejoe91

pykilosort/Dockerfile

@@ -0,0 +1,54 @@
+FROM nvidia/cuda:10.0-base-ubuntu18.04 
+
+LABEL maintainer="Vincent Prevosto <[email protected]>"
+
+# USER root
+# Ubuntu package installs
+RUN apt update && \
+    apt install -y --no-install-recommends \
+    libfftw3-dev \
+    git \
+    wget && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# install miniconda
+ENV MINICONDA_VERSION 4.12.0
+ENV CONDA_DIR /home/miniconda3
+ENV LATEST_CONDA_SCRIPT "Miniconda3-py38_$MINICONDA_VERSION-Linux-x86_64.sh"
+
+RUN wget --quiet https://repo.anaconda.com/miniconda/$LATEST_CONDA_SCRIPT -O ~/miniconda.sh && \
+    bash ~/miniconda.sh -b -p $CONDA_DIR && \
+    rm ~/miniconda.sh
+ENV PATH=$CONDA_DIR/bin:$PATH
+RUN conda update conda && \
+    conda install conda-build
+
+# make conda activate command available from /bin/bash --login shells
+RUN echo ". $CONDA_DIR/etc/profile.d/conda.sh" >> /root/.profile 
+# make conda activate command available from /bin/bash --interactive shells
+RUN conda init bash
+
+# Install python port of pykilosort
+RUN git clone -b drift_test_stable https://github.com/kushbanga/pykilosort.git /src/pykilosort
+WORKDIR /src/pykilosort
+# Switch to MouseLand/develop branch - may revert to kushbanga/drift_test_stable in the future
+RUN git remote add MouseLand https://github.com/MouseLand/pykilosort.git && \
+    git fetch MouseLand && git checkout MouseLand/develop && git checkout -b develop
+
+# modify env file so that env extends on base
+RUN sed -i "s/pyks2/base/" pyks2.yml
+
+# Create environment
+RUN conda env update --name base --file pyks2.yml --prune
+
+# Install pykilosort
+RUN conda install --quiet --yes ipykernel && \
+    python -m ipykernel install --user --display-name "pyKilosort" && \ 
+    conda develop .   
+
+# install phylib from master because of read-only bug (https://github.com/cortex-lab/phylib/pull/35)
+RUN pip uninstall -y phylib && pip install git+https://github.com/cortex-lab/phylib.git
+
+#--name pyks2
+WORKDIR /

pykilosort/README.md

@@ -0,0 +1,39 @@
+### Build this image
+Build default image: 
+docker build -t pykilosort:latest -f dockerfiles/Dockerfile .
+
+Build image with additional testing components: 
+docker build -t pykilosort:test -f dockerfiles/Dockerfile.testing context
+
+### Run container
+docker run --rm -it -v <host-data-folder>:<docker-data-folder> --gpus all pykilosort:latest
+flags:
+--rm: removes container once it's stopped
+-it: for interactive session
+-v: mounted volumes (directories)
+--gpus: enables GPU use within container 
+
+### Test
+Download Neuropixel 1.0 data to your data directory: https://catalystneuro.github.io/spike-sorting-hackathon/datasets/datasets.html#allen-institute-example
+(see also https://github.com/int-brain-lab/pykilosort/tree/ibl_prod/examples, although apparently not up to date)
+
+$docker run --rm -it -v /my/dir/to/data:/data --gpus all pykilosort
+#conda activate pyks2
+#cd /data
+    --- alternatively, get data from kachery (if installed and configured) ---
+    ```
+    wget https://catalystneuro.github.io/spike-sorting-hackathon/datasets/examples/example_allen_NP1.py
+    python example_allen_NP1.py
+    ```
+
+The either run tests in ipython console, or run example from /home/test_file directory (if using `.testing` image), after editing directory paths
+#ipython
+```
+from pathlib import Path
+from pykilosort import run, add_default_handler, np1_probe, np2_probe
+
+data_path = Path('/data/Allen_Institute_NP1/continuous_1min.bin')
+dir_path = Path('/data/Allen_Institute_NP1/output')
+add_default_handler(level='INFO') # print output as the algorithm runs
+run(data_path, dir_path=dir_path, probe=np1_probe(sync_channel=False))
+```

pykilosort/build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker build -t spikeinterface/pykilosort-base:latest -t spikeinterface/pykilosort-base:0.1.0 .

pykilosort/context/packages/testing_requirements.txt

@@ -0,0 +1,5 @@
+spikeinterface[full]
+dandi
+kachery
+kachery-cloud
+git+https://github.com/flatironinstitute/spikeforest2.git@master

pykilosort/context/test_files/example_general_users.py

@@ -0,0 +1,19 @@
+from pathlib import Path
+from pykilosort import run, add_default_handler, np1_probe, np2_probe
+
+# Run standard ks2.5 algorithm for a np1 probe
+data_path = Path('path/to/data/data.bin')
+dir_path = Path('path/to/output/folder') # by default uses the same folder as the dataset
+add_default_handler(level='INFO') # print output as the algorithm runs
+run(data_path, dir_path=dir_path, probe=np1_probe())
+
+# Run chronic recordings for a np2 probe
+# For now this still uses ks2.5 clustering, chronic clustering algorithm coming soon!
+data_paths = [
+    Path('path/to/first/dataset/dataset.bin'),
+    Path('path/to/second/dataset/dataset.bin'),
+    Path('path/to/third/dataset/dataset.bin'),
+]
+dir_path = Path('path/to/output/folder') # by default uses the same folder as the first dataset
+add_default_handler(level='INFO')
+run(data_paths, dir_path=dir_path, probe=np2_probe(), low_memory=True)

pykilosort/push.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker push --all-tags spikeinterface/pykilosort-base