update dev scripts

Author: stemangiola

dev/cellxgene_to_metadata.R

@@ -977,38 +977,38 @@ job::job({
 
   # Perform optimised joins within DuckDB
   copy_query <- "
-  COPY (
-    SELECT 
-      cell_to_refined_sample_from_Mengyuan.cell_,
-      cell_to_refined_sample_from_Mengyuan.observation_joinid,
-      cell_to_refined_sample_from_Mengyuan.dataset_id,
-      cell_to_refined_sample_from_Mengyuan.sample_id,
-      cell_to_refined_sample_from_Mengyuan.cell_type,
-      cell_to_refined_sample_from_Mengyuan.cell_type_ontology_term_id,
-      sample_metadata.*,
-      age_days_tbl.age_days,
-      tissue_grouped.tissue_groups
-    
-    FROM cell_to_refined_sample_from_Mengyuan
-    
-    LEFT JOIN cell_ids_for_metadata
-      ON cell_ids_for_metadata.cell_ = cell_to_refined_sample_from_Mengyuan.cell_
-      AND cell_ids_for_metadata.observation_joinid = cell_to_refined_sample_from_Mengyuan.observation_joinid
-      AND cell_ids_for_metadata.dataset_id = cell_to_refined_sample_from_Mengyuan.dataset_id
-      
-    LEFT JOIN sample_metadata
-      ON cell_ids_for_metadata.sample_ = sample_metadata.sample_
-      AND cell_ids_for_metadata.donor_id = sample_metadata.donor_id
-      AND cell_ids_for_metadata.dataset_id = sample_metadata.dataset_id
-      
-    LEFT JOIN age_days_tbl
-      ON age_days_tbl.development_stage = sample_metadata.development_stage
- 
-    LEFT JOIN tissue_grouped
-      ON tissue_grouped.tissue = sample_metadata.tissue
-      
-  ) TO '/vast/projects/cellxgene_curated/metadata_cellxgenedp_Apr_2024/cell_metadata_new.parquet'
-  (FORMAT PARQUET, COMPRESSION 'gzip');
+COPY (
+  SELECT 
+    cell_to_refined_sample_from_Mengyuan.cell_,
+    cell_to_refined_sample_from_Mengyuan.observation_joinid,
+    cell_to_refined_sample_from_Mengyuan.dataset_id,
+    cell_to_refined_sample_from_Mengyuan.sample_id,
+    cell_to_refined_sample_from_Mengyuan.cell_type,
+    cell_to_refined_sample_from_Mengyuan.cell_type_ontology_term_id,
+    sample_metadata.*,
+    age_days_tbl.age_days,
+    tissue_grouped.tissue_groups
+  
+  FROM cell_to_refined_sample_from_Mengyuan
+  
+  LEFT JOIN cell_ids_for_metadata
+    ON cell_ids_for_metadata.cell_ = cell_to_refined_sample_from_Mengyuan.cell_
+    AND cell_ids_for_metadata.observation_joinid = cell_to_refined_sample_from_Mengyuan.observation_joinid
+    AND cell_ids_for_metadata.dataset_id = cell_to_refined_sample_from_Mengyuan.dataset_id
+    
+  LEFT JOIN sample_metadata
+    ON cell_ids_for_metadata.sample_ = sample_metadata.sample_
+    AND cell_ids_for_metadata.donor_id = sample_metadata.donor_id
+    AND cell_ids_for_metadata.dataset_id = sample_metadata.dataset_id
+    
+  LEFT JOIN age_days_tbl
+    ON age_days_tbl.development_stage = sample_metadata.development_stage
+
+  LEFT JOIN tissue_grouped
+    ON tissue_grouped.tissue = sample_metadata.tissue
+    
+) TO '/vast/projects/cellxgene_curated/metadata_cellxgenedp_Apr_2024/cell_metadata.parquet'
+(FORMAT PARQUET, COMPRESSION 'gzip');
 "
 
   # Execute the final query to write the result to a Parquet file
@@ -1024,7 +1024,7 @@ job::job({
 
 cell_metadata = tbl(
   dbConnect(duckdb::duckdb(), dbdir = ":memory:"),
-  sql("SELECT * FROM read_parquet('/vast/projects/cellxgene_curated/metadata_cellxgenedp_Apr_2024/cell_metadata_new.parquet')")
+  sql("SELECT * FROM read_parquet('/vast/projects/cellxgene_curated/metadata_cellxgenedp_Apr_2024/cell_metadata.parquet')")
 ) 
 
 tissues_grouped = get_tissue_grouped() 
@@ -1081,7 +1081,7 @@ dbDisconnect(con, shutdown = TRUE)
 non_immune_harmonisation = 
   read_csv("/vast/projects/mangiola_immune_map/PostDoc/CuratedAtlasQueryR/dev/cell_type_harmonisation_non_immune.csv") 
 
-system("~/bin/rclone copy /vast/projects/mangiola_immune_map/PostDoc/CuratedAtlasQueryR/dev/cell_type_harmonisation_non_immune.csv box_adelaide:/Mangiola_ImmuneAtlas/reannotation_consensus/")
+# system("~/bin/rclone copy /vast/projects/mangiola_immune_map/PostDoc/CuratedAtlasQueryR/dev/cell_type_harmonisation_non_immune.csv box_adelaide:/Mangiola_ImmuneAtlas/reannotation_consensus/")
 
 
 tbl(

dev/execute_hpcell_on_census_and_defining_data_tranformation.R

@@ -218,19 +218,19 @@ tar_script({
   library(crew.cluster)
   tar_option_set(
     memory = "transient", 
-    garbage_collection = 100, 
+    garbage_collection = 1000, 
     storage = "worker", 
     retrieval = "worker", 
     error = "continue", 
-    #  debug = "dataset_id_sce_b5312463451d7ee3", 
+    debug = "annotation_tbl_light", 
     cue = tar_cue(mode = "never"), 
     controller = crew_controller_group(
       list(
         crew_controller_slurm(
           name = "tier_1", 
           script_lines = "#SBATCH --mem 8G",
           slurm_cpus_per_task = 1, 
-          workers = 300, 
+          workers = 500, 
           tasks_max = 10,
           verbose = T,
           launch_max = 5
@@ -276,7 +276,8 @@ tar_script({
       unnest(blueprint_scores_fine) |> 
       select(.cell, blueprint_first.labels.fine, monaco_first.labels.fine, any_of("azimuth_predicted.celltype.l2"), monaco_scores_fine, contains("macro"), contains("CD4") ) |> 
       unnest(monaco_scores_fine) |> 
-      select(.cell, blueprint_first.labels.fine, monaco_first.labels.fine, any_of("azimuth_predicted.celltype.l2"), contains("macro") , contains("CD4"), contains("helper"), contains("Th")) 
+      select(.cell, blueprint_first.labels.fine, monaco_first.labels.fine, any_of("azimuth_predicted.celltype.l2"), contains("macro") , contains("CD4"), contains("helper"), contains("Th")) |> 
+      rename(cell_ = .cell)
   }
 
   list(
@@ -313,7 +314,7 @@ job::job({
   tar_make(
     script = "/vast/scratch/users/mangiola.s/lighten_annotation_tbl_target.R", 
     store = "/vast/scratch/users/mangiola.s/lighten_annotation_tbl_target", 
-    reporter = "summary"
+    reporter = "summary", callr_function = NULL
   )
 
 })
@@ -323,16 +324,26 @@ library(arrow)
 library(dplyr)
 library(duckdb)
 
-cell_type_original <- tbl(
+# Write annotation light
+tar_read(annotation_tbl_light, store = "/vast/scratch/users/mangiola.s/lighten_annotation_tbl_target") |> 
+  rename(
+    blueprint_first_labels_fine = blueprint_first.labels.fine, 
+    monaco_first_labels_fine = monaco_first.labels.fine, 
+    azimuth_predicted_celltype_l2 = azimuth_predicted.celltype.l2
+  ) |> 
+  write_parquet("/vast/projects/cellxgene_curated/metadata_cellxgenedp_Apr_2024/annotation_tbl_light.parquet")
+
+cell_metadata <- tbl(
   dbConnect(duckdb::duckdb(), dbdir = ":memory:"),
   sql("SELECT * FROM read_parquet('/vast/projects/cellxgene_curated/metadata_cellxgenedp_Apr_2024/cell_metadata.parquet')")
 ) |>
   mutate(cell_ = paste0(cell_, "___", dataset_id)) |> 
-  select(.cell = cell_, observation_joinid, contains("cell_type"), dataset_id,  self_reported_ethnicity, tissue, donor_id,  sample_id, is_primary_data, assay)
+  select(cell_, observation_joinid, contains("cell_type"), dataset_id,  self_reported_ethnicity, tissue, donor_id,  sample_id, is_primary_data, assay)
+
 
 
 tar_read(annotation_tbl_light, store = "/vast/scratch/users/mangiola.s/lighten_annotation_tbl_target") |>
-  left_join(cell_type_original, copy = TRUE) |> 
+  left_join(cell_metadata, copy = TRUE) |> 
   write_parquet("/vast/projects/cellxgene_curated/metadata_cellxgenedp_Apr_2024/cell_annotation.parquet")
 
 system("~/bin/rclone copy /vast/projects/cellxgene_curated/metadata_cellxgenedp_Apr_2024/cell_annotation.parquet box_adelaide:/Mangiola_ImmuneAtlas/reannotation_consensus/")