Training refactor (#37)

* release candidate initial commit

* fix docstrings and formatting

---------

Co-authored-by: Jason Vander Heiden <[email protected]>

Author: tony-kuo

.github/workflows/docker.yaml

@@ -34,7 +34,7 @@ jobs:
       - name: Build and push Docker image
         uses: docker/build-push-action@v4
         with:
-          context: ./docker
+          context: .
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.github/workflows/pages.yaml

@@ -48,4 +48,4 @@ jobs:
     steps:
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v4
+        uses: actions/deploy-pages@v4

MANIFEST.in

@@ -1,3 +1,4 @@
 exclude .gitignore
 recursive-exclude .github *
 recursive-exclude docker *
+recursive-exclude scripts *

NEWS.rst

@@ -1,6 +1,29 @@
 Release Notes
 ================================================================================
 
+Version 0.4.0:  May 05, 2025
+--------------------------------------------------------------------------------
+
+General:
+  + A new training tutorial has been added which describes the new training
+    workflow. This includes data preparation, training, and post-training data
+    structures using the new scripts.
+
+Training:
+  + A new training workflow has been added to use CellArr, a TileDB based
+    framework, as the data store to streamline the end-to-end process. This
+    replaces the old Zarr based workflows.
+  + New data loaders and samplers for CellArr data have been added in the
+    ``tiledb_data_models`` module.
+  + A example training script has been added to show how to train models as
+    ``scripts/train.py``.
+  + New scripts for creating all post-training data structures have been added
+    in the folder ``scripts``.
+  + New utility methods that make use of the CellArr store:
+    ``utils.query_tiledb_df`` to query a tiledb dataframe,
+    ``utils.adata_from_tiledb`` to extract cells from the tiledb stores based on
+    index, including raw counts.
+
 Version 0.3.0:  November 19, 2024
 --------------------------------------------------------------------------------
 

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM fedora:38
+FROM fedora:42
 LABEL maintainer="Jason Anthony Vander Heiden [[email protected]]" \
       org.opencontainers.image.description="SCimilarity" \
       org.opencontainers.image.source="https://github.com/genentech/scimilarity"
@@ -10,7 +10,7 @@ VOLUME /workspace
 VOLUME /scratch
 
 # Tools
-COPY start-notebook.sh /usr/local/bin/start-notebook
+COPY docker/start-notebook.sh /usr/local/bin/start-notebook
 
 # Environment
 ENV SCDATA_HOME=/data
@@ -28,13 +28,13 @@ RUN dnf -y update && dnf install -y \
     python3 \
     python3-aiohttp \
     python3-asciitree \
-    python3-bash-kernel \
     python3-biopython \
     python3-cloudpickle \
     python3-Cython \
     python3-numcodecs \
     python3-dask \
     python3-dask+array \
+    python3-devel \
     python3-fasteners \
     python3-GitPython \
     python3-h5py \
@@ -74,6 +74,8 @@ RUN pip3 install \
 	scikit-misc \
 	numba \
 	tiledb \
+    tiledb-cloud \
+    tiledb-vector-search \
 	leidenalg \
 	louvain \
 	umap-learn \
@@ -83,7 +85,8 @@ RUN pip3 install \
 	captum \
 	torch \
 	pytorch-lightning \
-    scanpy
+    scanpy \
+    scrublet
 
 # Install SCimilarity API
 RUN git clone https://github.com/Genentech/scimilarity.git /tmp/scimilarity \

docs/api.rst

@@ -31,6 +31,7 @@ API Reference
 
     modules/anndata_data_models
     modules/nn_models
+    modules/tiledb_data_models
     modules/training_models
     modules/triplet_selector
     modules/zarr_data_models
@@ -69,6 +70,7 @@ support for these training tasks.
 
 * :mod:`scimilarity.anndata_data_models`
 * :mod:`scimilarity.nn_models`
+* :mod:`scimilarity.tiledb_data_models`
 * :mod:`scimilarity.training_models`
 * :mod:`scimilarity.triplet_selector`
 * :mod:`scimilarity.zarr_data_models`

docs/install.rst

@@ -48,6 +48,7 @@ You can download the following pretrained models for use with SCimilarity from
 Zenodo:
 https://zenodo.org/records/10685499
 
+
 Conda environment setup
 --------------------------------------------------------------------------------
 

docs/modules/tiledb_data_models.rst

@@ -0,0 +1,6 @@
+scimilarity.tiledb_data_models
+--------------------------------------------------------------------------------
+
+.. automodule:: scimilarity.tiledb_data_models
+    :members:
+    :show-inheritance: