Merge pull request #8 from SingleRust/feature-dev-load-speedup

Added concat framework, still work in progress

Author: ianfd

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "anndata-memory"
-version = "1.0.5"
+version = "1.0.6"
 edition = "2021"
 readme = "README.md"
 repository = "https://github.com/SingleRust/Anndata-Memory"

src/concat/csr.rs

@@ -0,0 +1,23 @@
+use std::path::Path;
+
+use crate::{ConcatStrategy, LoadingStrategy};
+
+pub(crate) mod csr;
+
+// TODO!
+
+
+pub fn concat_datasets_backed_to_memory<P: AsRef<Path>>(paths: &[P], load_strategy: Option<LoadingStrategy>, merge_strategy: Option<ConcatStrategy>) {
+
+    // general approach have dataset:
+    // 1. collect statistics: ensure all datasets have same datatype (for sparse datatypes collect num rows, num cols, nnz)
+    // 2. delegate to CSR, CSC datatype
+    // 3. load sequentially one dataset after another, if chunked approach, use anndata-rs to handle chunking 0
+
+
+
+
+
+}
+
+

src/concat/mod.rs

@@ -5,6 +5,7 @@ pub(crate) mod utils;
 pub(crate) mod chunked_loader;
 pub(crate) mod optimized_loader;
 mod loader;
+mod concat;
 
 pub use ad::IMAnnData;
 pub use ad::helpers::IMArrayElement;
@@ -17,6 +18,8 @@ pub use converter::convert_to_backed;
 pub use converter::convert_to_new_backed_h5;
 pub use base::DeepClone;
 
+
+
 #[derive(Clone, Debug)]
 pub enum LoadingStrategy {
     Auto,           
@@ -43,4 +46,12 @@ impl Default for LoadingConfig {
     }
 }
 
+#[derive(Clone, Debug)]
+pub enum ConcatStrategy {
+    ConcatObs,
+    ConcatVars,
+    Union,
+    Intersection
+}
+
 pub use loader::{load_h5ad, load_h5ad_fast, load_h5ad_conservative, load_h5ad_with_config};