Add support for unharmonised metadata

Author: multimeric

NAMESPACE

@@ -4,6 +4,7 @@ S3method(as.sparse,DelayedMatrix)
 export(get_SingleCellExperiment)
 export(get_metadata)
 export(get_seurat)
+export(get_unharmonised_metadata)
 import(Seurat)
 import(dbplyr)
 importFrom(BiocGenerics,cbind)

R/dev.R

@@ -1,31 +1,21 @@
 # Utility scripts for development purposes, that are not exported to users
 
-#' Update the metadata database in nectar using a newly created data frame
-#' @param metadata The data frame to upload
-#' @param version The version for the new metadata as a character scalar, e.g.
-#'   "0.2.3"
+#' Upload a file to the Nectar object store
+#' @param source A character scalar indicating the local path to the file to
+#'   upload
+#' @param container A character scalar indicating the name of the container to
+#'   upload to
+#' @param name An optional character scalar indicating the name the file should
+#'   have after being uploaded. Defaults to being the basename of the source
+#'   file.
 #' @param credential_id The OpenStack application credential ID as a character
 #'   scalar. This is optional because you can alternatively source a
 #'   `-openrc.sh` file instead of providing it here.
 #' @param credential_id The OpenStack application credential secret as a
 #'   character scalar
-#' @noRd
-#' @example 
-#' \dontrun{
-#'  metadata = CuratedAtlasQueryR::get_metadata() |> head(10) |> dplyr::collect()
-#'  update_database(metadata, "0.2.3", "rfypdlunhrfopdnkrs", "3q5lw3qntafptdfsrdh-wa4p8h")
-#'  # Prints "metadata.0.2.3.parquet" if successful
-#' }
-update_database = function(metadata, version, credential_id = NULL, credential_secret = NULL){
-    # These are optional dev packages
-    rlang::check_installed(c("arrow", "glue", "basilisk"))
-    
-    # Create parquet
-    dir <- tempdir()
-    parquet_name <- glue::glue("metadata.{version}.parquet")
-    parquet_path <- file.path(dir, parquet_name)
-    arrow::write_parquet(metadata, sink=parquet_path)
-    
+#' @return NULL
+#' @keywords internal
+upload_swift = function(source, container, name = basename(source), credential_id = NULL, credential_secret = NULL){
     # Create the basilisk environment
     swift_env <- basilisk::BasiliskEnvironment(
         envname="swift-nectar-upload",
@@ -57,13 +47,54 @@ update_database = function(metadata, version, credential_id = NULL, credential_s
         "06d6e008e3e642da99d806ba3ea629c5",
         auth,
         "upload",
-        "metadata",
-        parquet_path,
+        container,
+        source,
         "--object-name",
-        parquet_name
+        name
     )
 
     # Perform the upload
     system2(reticulate::py_exe(), args=args)
     basilisk::basiliskStop(proc)
+    
+    invisible(NULL)
+}
+
+#' Update the metadata database in nectar using a newly created data frame
+#' @param metadata The data frame to upload
+#' @param version The version for the new metadata as a character scalar, e.g.
+#'   "0.2.3"
+#' @inheritDotParams upload_swift
+#' @example 
+#' \dontrun{
+#'  metadata = CuratedAtlasQueryR::get_metadata() |> head(10) |> dplyr::collect()
+#'  update_database(metadata, "0.2.3", credential_id = "ABCDEFGHIJK", credential_secret = "ABCD1234EFGH-5678IJK")
+#'  # Prints "metadata.0.2.3.parquet" if successful
+#' }
+#' @keywords internal
+update_database = function(metadata, version, ...){
+    # These are optional dev packages
+    rlang::check_installed(c("arrow", "glue", "basilisk"))
+    
+    dir <- tempdir()
+    parquet_name <- glue::glue("metadata.{version}.parquet")
+    parquet_path <- file.path(dir, parquet_name)
+    arrow::write_parquet(metadata, sink=parquet_path)
+    
+    upload_swift(parquet_path, container="metadata", name=parquet_name, ...)
+}
+
+#' Update the unharmonised parquet files
+#' @param unharmonised_parquet_dir The path to a directory containing parquet
+#'   files, one for each dataset, e.g.
+#'   /vast/projects/cellxgene_curated/metadata_non_harmonised_parquet_0.2
+#' @inheritDotParams upload_swift
+#' @keywords internal
+#' @examples
+#' \dontrun{
+#' update_unharmonised("/vast/projects/cellxgene_curated/metadata_non_harmonised_parquet_0.2", credential_id = "ABCDEFGHIJK", credential_secret = "ABCD1234EFGH-5678IJK")
+#' }
+update_unharmonised = function(unharmonised_parquet_dir, ...){
+    # name="/" forces it have no prefix, ie be at the top level in the bucket
+    upload_swift(unharmonised_parquet_dir, container="unharmonised_metadata", name="/", ...)
 }

R/query.R

@@ -448,3 +448,41 @@ get_metadata <- function(
         dbConnect(drv = _, read_only = TRUE) |>
         tbl(db_path)
 }
+
+#' Returns unharmonised metadata for selected datasets.
+#' 
+#' Various metadata fields are *not* common between datasets, so it does not make
+#' sense for these to live in the main metadata table. This function is a 
+#' utility that allows easy fetching of this data if necessary.
+#'
+#' @param dataset_ids A character vector, where each entry is a dataset ID
+#'  obtained from the `$dataset_id` column of the table returned from
+#'  [get_metadata()]
+#' @return
+#' @export
+#' @examples
+#' dataset_id = "838ea006-2369-4e2c-b426-b2a744a2b02b"
+#' harmonised_meta = get_metadata() |> dplyr::filter(dataset_id = dataset_id)
+#' unharmonised_meta = get_unharmonised_metadata(dataset_id)
+#' unharmonised_tbl = unharmonised_meta[[dataset_id]]
+#' dplyr::join(harmonised_meta, unharmonised_tbl, by=c("dataset_id", "cell_"))
+get_unharmonised_metadata = function(
+        dataset_ids,
+        remote_url = "https://object-store.rc.nectar.org.au/v1/AUTH_06d6e008e3e642da99d806ba3ea629c5/unharmonised_metadata",
+        cache_directory = get_default_cache_dir()
+        ){
+    unharmonised_root <- file.path(cache_directory, COUNTS_VERSION, "unharmonised")
+    duck = duckdb() |> dbConnect(drv = _, read_only = TRUE)
+    dataset_ids |> 
+        purrr::set_names() |>
+        purrr::map(function(dataset_id){
+            file_name = glue::glue("{dataset_id}.parquet")
+            local_path = file.path(unharmonised_root, file_name)
+            glue::glue("{remote_url}/{file_name}") |>
+                sync_remote_file(
+                local_path,
+                progress(type = "down", con = stderr())
+            )
+            tbl(duck, local_path)
+        })
+}

tests/testthat/test-query.R

@@ -156,3 +156,19 @@ test_that("get_SingleCellExperiment() assigns the right cell ID to each cell", {
         assay_2[, colnames(assay_1)]
     )
 })
+
+test_that("get_unharmonised_metadata works with one ID", {
+    dataset_id = "838ea006-2369-4e2c-b426-b2a744a2b02b"
+    unharmonised_meta = get_unharmonised_metadata(dataset_id)
+    unharmonised_tbl = unharmonised_meta[[dataset_id]]
+    
+    expect_s3_class(unharmonised_meta, "list")
+    expect_s3_class(unharmonised_tbl, "tbl")
+})
+
+test_that("get_unharmonised_metadata works with multiple IDs", {
+    dataset_ids = c("838ea006-2369-4e2c-b426-b2a744a2b02b", "83b9cb97-9ee4-404d-8cdf-ccede8235356.parquet")
+    unharmonised_meta = get_unharmonised_metadata(dataset_ids)
+    
+    expect_equal(names(unharmonised_meta), dataset_ids)
+})