clean up processing dataset component and test resource script

Author: rcannood

scripts/create_resources/test_resources.sh

@@ -19,66 +19,6 @@ DATASET_DIR=resources_test/task_cyto_batch_integration/starter_file
 
 mkdir -p $DATASET_DIR
 
-# TODO: get original_dataset.h5ad from somewhere
-
-# wget https://zenodo.org/records/13928969/files/ID1_Panel1_TP1.fcs?download=1 \
-#   -O $DATASET_DIR/ID1_Panel1_TP1.fcs
-
-# python << HERE
-# import readfcs
-# ad = readfcs.read("$DATASET_DIR/ID1_Panel1_TP1.fcs")
-# ad.layers["transformed"] = ad.X
-# del ad.X
-# # todo: add other preprocessing steps to make sure the dataset is a common dataset
-# ad.write_h5ad("$DATASET_DIR/common_dataset.h5ad")
-# HERE
-
-python << HERE
-import anndata as ad
-
-adata = ad.read_h5ad("resources_test/task_cyto_batch_integration/starter_file/common_dataset.h5ad")
-
-channelsofinterest = ['UV379-A',
- 'UV515-A',
- 'UV610-A',
- 'UV735-A',
- 'V431-A',
- 'V525-A',
- 'V586-A',
- 'V605-A',
- 'V677-A',
- 'V710-A',
- 'V750-A',
- 'V810-A',
- 'B530-A',
- 'B710-A',
- 'YG586-A',
- 'YG610-A',
- 'YG670-A',
- 'YG780-A',
- 'R670-A',
- 'R730-A']
-adata.var.rename(columns={"n":"numeric_id"}, inplace=True)
-marker_types = ["lineage" if chan in channelsofinterest else 'functional' for chan in adata.var["channel"]]
-to_correct = [True if chan in channelsofinterest else False for chan in adata.var["channel"]]
-adata.var["marker_type"] = marker_types
-adata.var['to_correct'] = to_correct
-adata.uns['dataset_id'] = 'XXXXX'
-adata.uns['dataset_name'] = 'Summer School data'
-adata.uns['dataset_summary'] = 'Draft data for cytometry batch integration benchmark'
-adata.uns['dataset_description'] = '''
-This is a draft dataset for the cytometry batch integration benchmark (Summer School). 
-It contains only samples from one batch (Day1). 
-Even though a preprocessed layer is available, it only contains arcsinh transformed data (not cleaned or compensated data).
-'''
-adata.uns['dataset_url'] = "https://saeyslab.sites.vib.be"
-adata.uns['dataset_organism'] = "mus_musculus"
-adata.uns['dataset_reference'] = "unpublished"
-
-out_file = "resources_test/task_cyto_batch_integration/starter_file/common_dataset.h5ad"
-adata.write_h5ad(out_file, compression="gzip")
-HERE
-
 # process dataset
 viash run src/data_processors/process_dataset/config.vsh.yaml -- \
   --input $RAW_DATA/common_dataset.h5ad \

src/data_processors/process_dataset/config.vsh.yaml

@@ -1,10 +1,6 @@
 __merge__: ../../api/comp_data_processor.yaml
 name: process_dataset
-arguments:
-  - name: "--validation_sample_names"
-    type: "string"
-    description: "The process method to assign train/test."
-    multiple: true
+
 resources:
   - type: python_script
     path: script.py

src/data_processors/process_dataset/script.py

@@ -5,10 +5,9 @@
 ## VIASH START
 par = {
     'input': 'resources_test/task_cyto_batch_integration/starter_file/common_dataset.h5ad',
-    'validation_sample_names': [],
-    'output_unintegrated': 'unintegrated.h5ad',
-    'output_unintegrated_censored': 'unintegrated_censored.h5ad',
-    'output_validation': 'validation.h5ad'
+    'output_unintegrated': 'resources_test/task_cyto_batch_integration/starter_file/unintegrated.h5ad',
+    'output_unintegrated_censored': 'resources_test/task_cyto_batch_integration/starter_file/unintegrated_censored.h5ad',
+    'output_validation': 'resources_test/task_cyto_batch_integration/starter_file/validation.h5ad'
 }
 meta = {
     'resources_dir': 'target/executable/data_processors/process_dataset',