Rename dataset loader and add compression argument to more components (#930)

* rename component and workflow

* add output_compression

Author: rcannood

src/datasets/api/comp_dataset_loader.yaml

@@ -12,4 +12,9 @@ arguments:
     __merge__: file_raw.yaml
     direction: "output"
     required: true
+  - name: --output_compression
+    type: string
+    choices: [gzip, lzf]
+    required: false
+    example: gzip
 test_resources: []

src/datasets/api/comp_normalization.yaml

@@ -16,6 +16,11 @@ arguments:
     __merge__: file_normalized.yaml
     direction: output
     required: true
+  - name: --output_compression
+    type: string
+    choices: [gzip, lzf]
+    required: false
+    example: gzip
   - name: "--normalization_id"
     type: string
     description: "The normalization id to store in the dataset metadata. If not specified, the functionality name will be used."

src/datasets/api/comp_processor_hvg.yaml

@@ -20,6 +20,11 @@ arguments:
     direction: output
     __merge__: file_hvg.yaml
     required: true
+  - name: --output_compression
+    type: string
+    choices: [gzip, lzf]
+    required: false
+    example: gzip
   - name: "--var_hvg"
     type: string
     default: "hvg"

src/datasets/api/comp_processor_knn.yaml

@@ -20,6 +20,11 @@ arguments:
     direction: output
     __merge__: file_knn.yaml
     required: true
+  - name: --output_compression
+    type: string
+    choices: [gzip, lzf]
+    required: false
+    example: gzip
   - name: "--key_added"
     type: string
     default: "knn"

src/datasets/api/comp_processor_pca.yaml

@@ -24,6 +24,11 @@ arguments:
     direction: output
     __merge__: file_pca.yaml
     required: true
+  - name: --output_compression
+    type: string
+    choices: [gzip, lzf]
+    required: false
+    example: gzip
   - name: "--obsm_embedding"
     type: string
     default: "X_pca"

src/datasets/api/comp_processor_subset.yaml

@@ -22,6 +22,11 @@ arguments:
     __merge__: file_common_dataset.yaml
     direction: output
     required: false
+  - name: --output_compression
+    type: string
+    choices: [gzip, lzf]
+    required: false
+    example: gzip
 test_resources:
   - path: /resources_test/common/pancreas
     dest: resources_test/common/pancreas

src/datasets/api/comp_processor_svd.yaml

@@ -28,6 +28,11 @@ arguments:
     direction: output
     __merge__: file_svd.yaml
     required: false
+  - name: --output_compression
+    type: string
+    choices: [gzip, lzf]
+    required: false
+    example: gzip
   - name: "--obsm_embedding"
     type: string
     default: "X_svd"

src/datasets/loaders/scrnaseq/op3/config.vsh.yaml

@@ -1,4 +1,4 @@
-name: op3
+name: openproblems_op3
 namespace: datasets/loaders/scrnaseq
 description: |
   "Loads and preprocesses the OP3 dataset from GEO accession GSE279945."
@@ -39,7 +39,7 @@ argument_groups:
       - name: "--dataset_id"
         type: string
         description: "Unique identifier for the dataset"
-        default: "op3"
+        default: "openproblems_op3"
       - name: "--dataset_name"
         type: string
         description: "Human-readable name for the dataset"

src/datasets/processors/hvg/script.py

@@ -32,5 +32,4 @@
 adata.var[par["var_hvg_score"]] = out['dispersions_norm'].values
 
 print(">> Writing data", flush=True)
-adata.write_h5ad(par['output'])
-
+adata.write_h5ad(par['output'], compression=par["output_compression"])

src/datasets/processors/knn/script.py

@@ -23,5 +23,5 @@
 )
 
 print(">> Writing data", flush=True)
-adata.write_h5ad(par['output'])
+adata.write_h5ad(par['output'], compression=par["output_compression"])