Merge pull request #97 from stemangiola/fix-95

Add support for unharmonised metadata

Author: stemangiola

NAMESPACE

@@ -4,6 +4,7 @@ S3method(as.sparse,DelayedMatrix)
 export(get_SingleCellExperiment)
 export(get_metadata)
 export(get_seurat)
+export(get_unharmonised_metadata)
 import(Seurat)
 import(dbplyr)
 importFrom(BiocGenerics,cbind)
@@ -49,6 +50,7 @@ importFrom(purrr,map)
 importFrom(purrr,map_int)
 importFrom(purrr,pmap_chr)
 importFrom(purrr,reduce)
+importFrom(purrr,set_names)
 importFrom(purrr,transpose)
 importFrom(rlang,.data)
 importFrom(stats,setNames)

R/dev.R

@@ -1,31 +1,21 @@
 # Utility scripts for development purposes, that are not exported to users
 
-#' Update the metadata database in nectar using a newly created data frame
-#' @param metadata The data frame to upload
-#' @param version The version for the new metadata as a character scalar, e.g.
-#'   "0.2.3"
+#' Upload a file to the Nectar object store
+#' @param source A character scalar indicating the local path to the file to
+#'   upload
+#' @param container A character scalar indicating the name of the container to
+#'   upload to
+#' @param name An optional character scalar indicating the name the file should
+#'   have after being uploaded. Defaults to being the basename of the source
+#'   file.
 #' @param credential_id The OpenStack application credential ID as a character
 #'   scalar. This is optional because you can alternatively source a
 #'   `-openrc.sh` file instead of providing it here.
 #' @param credential_id The OpenStack application credential secret as a
 #'   character scalar
-#' @noRd
-#' @example 
-#' \dontrun{
-#'  metadata = CuratedAtlasQueryR::get_metadata() |> head(10) |> dplyr::collect()
-#'  update_database(metadata, "0.2.3", "rfypdlunhrfopdnkrs", "3q5lw3qntafptdfsrdh-wa4p8h")
-#'  # Prints "metadata.0.2.3.parquet" if successful
-#' }
-update_database = function(metadata, version, credential_id = NULL, credential_secret = NULL){
-    # These are optional dev packages
-    rlang::check_installed(c("arrow", "glue", "basilisk"))
-    
-    # Create parquet
-    dir <- tempdir()
-    parquet_name <- glue::glue("metadata.{version}.parquet")
-    parquet_path <- file.path(dir, parquet_name)
-    arrow::write_parquet(metadata, sink=parquet_path)
-    
+#' @return NULL
+#' @keywords internal
+upload_swift = function(source, container, name = basename(source), credential_id = NULL, credential_secret = NULL){
     # Create the basilisk environment
     swift_env <- basilisk::BasiliskEnvironment(
         envname="swift-nectar-upload",
@@ -57,13 +47,54 @@ update_database = function(metadata, version, credential_id = NULL, credential_s
         "06d6e008e3e642da99d806ba3ea629c5",
         auth,
         "upload",
-        "metadata",
-        parquet_path,
+        container,
+        source,
         "--object-name",
-        parquet_name
+        name
     )
 
     # Perform the upload
     system2(reticulate::py_exe(), args=args)
     basilisk::basiliskStop(proc)
+    
+    invisible(NULL)
+}
+
+#' Update the metadata database in nectar using a newly created data frame
+#' @param metadata The data frame to upload
+#' @param version The version for the new metadata as a character scalar, e.g.
+#'   "0.2.3"
+#' @inheritDotParams upload_swift
+#' @examples
+#' \dontrun{
+#'  metadata = CuratedAtlasQueryR::get_metadata() |> head(10) |> dplyr::collect()
+#'  update_database(metadata, "0.2.3", credential_id = "ABCDEFGHIJK", credential_secret = "ABCD1234EFGH-5678IJK")
+#'  # Prints "metadata.0.2.3.parquet" if successful
+#' }
+#' @keywords internal
+update_database = function(metadata, version, ...){
+    # These are optional dev packages
+    rlang::check_installed(c("arrow", "glue", "basilisk"))
+    
+    dir <- tempdir()
+    parquet_name <- glue::glue("metadata.{version}.parquet")
+    parquet_path <- file.path(dir, parquet_name)
+    arrow::write_parquet(metadata, sink=parquet_path)
+    
+    upload_swift(parquet_path, container="metadata", name=parquet_name, ...)
+}
+
+#' Update the unharmonised parquet files
+#' @param unharmonised_parquet_dir The path to a directory containing parquet
+#'   files, one for each dataset, e.g.
+#'   /vast/projects/cellxgene_curated/metadata_non_harmonised_parquet_0.2
+#' @inheritDotParams upload_swift
+#' @keywords internal
+#' @examples
+#' \dontrun{
+#' update_unharmonised("/vast/projects/cellxgene_curated/metadata_non_harmonised_parquet_0.2", credential_id = "ABCDEFGHIJK", credential_secret = "ABCD1234EFGH-5678IJK")
+#' }
+update_unharmonised = function(unharmonised_parquet_dir, ...){
+    # name="/" forces it have no prefix, ie be at the top level in the bucket
+    upload_swift(unharmonised_parquet_dir, container="unharmonised_metadata", name="/", ...)
 }

R/query.R

@@ -448,3 +448,52 @@ get_metadata <- function(
         dbConnect(drv = _, read_only = TRUE) |>
         tbl(db_path)
 }
+
+#' Returns unharmonised metadata for selected datasets.
+#'
+#' Various metadata fields are *not* common between datasets, so it does not
+#' make sense for these to live in the main metadata table. This function is a
+#' utility that allows easy fetching of this data if necessary.
+#'
+#' @param dataset_ids A character vector, where each entry is a dataset ID
+#'   obtained from the `$file_id` column of the table returned from
+#'   [get_metadata()]
+#' @param remote_url Optional character vector of length 1. An HTTP URL pointing
+#'   to the root URL under which all the unharmonised dataset files are located.
+#' @param cache_directory Optional character vector of length 1. A file path on
+#'   your local system to a directory (not a file) that will be used to store
+#'   the unharmonised metadata files.
+#' @importFrom purrr map set_names
+#' @importFrom glue glue
+#' @importFrom DBI dbConnect
+#' @importFrom duckdb duckdb
+#' @importFrom dplyr tbl
+#' @return A named list, where each name is a dataset file ID, and each value is
+#'   a "lazy data frame", ie a `tbl`.
+#' @export
+#' @examples
+#' dataset = "838ea006-2369-4e2c-b426-b2a744a2b02b"
+#' harmonised_meta = get_metadata() |> dplyr::filter(file_id == dataset) |> dplyr::collect()
+#' unharmonised_meta = get_unharmonised_metadata(dataset)
+#' unharmonised_tbl = dplyr::collect(unharmonised_meta[[dataset]])
+#' dplyr::left_join(harmonised_meta, unharmonised_tbl, by=c("file_id", "cell_"))
+get_unharmonised_metadata = function(
+        dataset_ids,
+        remote_url = "https://object-store.rc.nectar.org.au/v1/AUTH_06d6e008e3e642da99d806ba3ea629c5/unharmonised_metadata",
+        cache_directory = get_default_cache_dir()
+        ){
+    unharmonised_root <- file.path(cache_directory, COUNTS_VERSION, "unharmonised")
+    duck = duckdb() |> dbConnect(drv = _, read_only = TRUE)
+    dataset_ids |> 
+        set_names() |>
+        map(function(dataset_id){
+            file_name = glue::glue("{dataset_id}.parquet")
+            local_path = file.path(unharmonised_root, file_name)
+            glue("{remote_url}/{file_name}") |>
+                sync_remote_file(
+                local_path,
+                progress(type = "down", con = stderr())
+            )
+            tbl(duck, local_path)
+        })
+}

README.Rmd

@@ -275,6 +275,30 @@ metadata |>
 knitr::include_graphics("man/figures/HLA_A_tissue_plot.png")
 ```
 
+## Obtain Unharmonised Metadata
+
+Various metadata fields are *not* common between datasets, so it does not
+make sense for these to live in the main metadata table. However, we can
+obtain it using the `get_unharmonised_metadata()` function.
+
+Note how this table has additional columns that are not in the normal metadata:
+
+```{r}
+dataset = "838ea006-2369-4e2c-b426-b2a744a2b02b"
+unharmonised_meta = get_unharmonised_metadata(dataset)
+unharmonised_tbl = dplyr::collect(unharmonised_meta[[dataset]])
+unharmonised_tbl
+```
+
+If we have metadata from the normal metadata table that is from a single dataset,
+we can even join this additional metadata into one big data frame:
+```{r}
+harmonised_meta = get_metadata() |> dplyr::filter(file_id == dataset) |> dplyr::collect()
+dplyr::left_join(harmonised_meta, unharmonised_tbl, by=c("file_id", "cell_"))
+```
+
+
+
 # Cell metadata
 
 Dataset-specific columns (definitions available at cellxgene.cziscience.com)

man/get_unharmonised_metadata.Rd

@@ -156,3 +156,19 @@ test_that("get_SingleCellExperiment() assigns the right cell ID to each cell", {
         assay_2[, colnames(assay_1)]
     )
 })
+
+test_that("get_unharmonised_metadata works with one ID", {
+    dataset_id = "838ea006-2369-4e2c-b426-b2a744a2b02b"
+    unharmonised_meta = get_unharmonised_metadata(dataset_id)
+    unharmonised_tbl = unharmonised_meta[[dataset_id]]
+    
+    expect_type(unharmonised_meta, "list")
+    expect_s3_class(unharmonised_tbl, "tbl")
+})
+
+test_that("get_unharmonised_metadata works with multiple IDs", {
+    dataset_ids = c("838ea006-2369-4e2c-b426-b2a744a2b02b", "83b9cb97-9ee4-404d-8cdf-ccede8235356")
+    unharmonised_meta = get_unharmonised_metadata(dataset_ids)
+    
+    expect_equal(names(unharmonised_meta), dataset_ids)
+})