WIP implementation of the original suggested API

Author: multimeric

NAMESPACE

@@ -29,9 +29,11 @@ importFrom(dplyr,as_tibble)
 importFrom(dplyr,collect)
 importFrom(dplyr,filter)
 importFrom(dplyr,full_join)
+importFrom(dplyr,group_by)
 importFrom(dplyr,inner_join)
 importFrom(dplyr,mutate)
 importFrom(dplyr,pull)
+importFrom(dplyr,summarise)
 importFrom(dplyr,tbl)
 importFrom(dplyr,tibble)
 importFrom(dplyr,transmute)

R/query.R

@@ -447,53 +447,4 @@ get_metadata <- function(
     duckdb() |>
         dbConnect(drv = _, read_only = TRUE) |>
         tbl(db_path)
-}
-
-#' Returns unharmonised metadata for selected datasets.
-#'
-#' Various metadata fields are *not* common between datasets, so it does not
-#' make sense for these to live in the main metadata table. This function is a
-#' utility that allows easy fetching of this data if necessary.
-#'
-#' @param dataset_ids A character vector, where each entry is a dataset ID
-#'   obtained from the `$file_id` column of the table returned from
-#'   [get_metadata()]
-#' @param remote_url Optional character vector of length 1. An HTTP URL pointing
-#'   to the root URL under which all the unharmonised dataset files are located.
-#' @param cache_directory Optional character vector of length 1. A file path on
-#'   your local system to a directory (not a file) that will be used to store
-#'   the unharmonised metadata files.
-#' @importFrom purrr map set_names
-#' @importFrom glue glue
-#' @importFrom DBI dbConnect
-#' @importFrom duckdb duckdb
-#' @importFrom dplyr tbl
-#' @return A named list, where each name is a dataset file ID, and each value is
-#'   a "lazy data frame", ie a `tbl`.
-#' @export
-#' @examples
-#' dataset = "838ea006-2369-4e2c-b426-b2a744a2b02b"
-#' harmonised_meta = get_metadata() |> dplyr::filter(file_id == dataset) |> dplyr::collect()
-#' unharmonised_meta = get_unharmonised_metadata(dataset)
-#' unharmonised_tbl = dplyr::collect(unharmonised_meta[[dataset]])
-#' dplyr::left_join(harmonised_meta, unharmonised_tbl, by=c("file_id", "cell_"))
-get_unharmonised_metadata = function(
-        dataset_ids,
-        remote_url = "https://object-store.rc.nectar.org.au/v1/AUTH_06d6e008e3e642da99d806ba3ea629c5/unharmonised_metadata",
-        cache_directory = get_default_cache_dir()
-        ){
-    unharmonised_root <- file.path(cache_directory, COUNTS_VERSION, "unharmonised")
-    duck = duckdb() |> dbConnect(drv = _, read_only = TRUE)
-    dataset_ids |> 
-        set_names() |>
-        map(function(dataset_id){
-            file_name = glue::glue("{dataset_id}.parquet")
-            local_path = file.path(unharmonised_root, file_name)
-            glue("{remote_url}/{file_name}") |>
-                sync_remote_file(
-                local_path,
-                progress(type = "down", con = stderr())
-            )
-            tbl(duck, local_path)
-        })
-}
+}

R/unharmonised.R

@@ -0,0 +1,71 @@
+#' Returns unharmonised metadata for selected datasets.
+#'
+#' Various metadata fields are *not* common between datasets, so it does not
+#' make sense for these to live in the main metadata table. This function is a
+#' utility that allows easy fetching of this data if necessary.
+#'
+#' @param dataset_ids A character vector, where each entry is a dataset ID
+#'   obtained from the `$file_id` column of the table returned from
+#'   [get_metadata()]
+#' @param remote_url Optional character vector of length 1. An HTTP URL pointing
+#'   to the root URL under which all the unharmonised dataset files are located.
+#' @param cache_directory Optional character vector of length 1. A file path on
+#'   your local system to a directory (not a file) that will be used to store
+#'   the unharmonised metadata files.
+#' @importFrom purrr map set_names
+#' @importFrom glue glue
+#' @importFrom DBI dbConnect
+#' @importFrom duckdb duckdb
+#' @importFrom dplyr tbl filter
+#' @return A named list, where each name is a dataset file ID, and each value is
+#'   a "lazy data frame", ie a `tbl`.
+#' @examples
+#' dataset = "838ea006-2369-4e2c-b426-b2a744a2b02b"
+#' harmonised_meta = get_metadata() |> dplyr::filter(file_id == dataset) |> dplyr::collect()
+#' unharmonised_meta = get_unharmonised_metadata_list(dataset)
+#' unharmonised_tbl = dplyr::collect(unharmonised_meta[[dataset]])
+#' dplyr::left_join(harmonised_meta, unharmonised_tbl, by=c("file_id", "cell_"))
+get_unharmonised_dataset = function(
+        dataset_id,
+        cells = NULL,
+        conn = duckdb() |> dbConnect(drv = _, read_only = TRUE),
+        remote_url = "https://object-store.rc.nectar.org.au/v1/AUTH_06d6e008e3e642da99d806ba3ea629c5/unharmonised_metadata",
+        cache_directory = get_default_cache_dir()
+){
+    unharmonised_root <- file.path(cache_directory, COUNTS_VERSION, "unharmonised")
+    file_name = glue::glue("{dataset_id}.parquet")
+    local_path = file.path(unharmonised_root, file_name)
+    glue("{remote_url}/{file_name}") |>
+        sync_remote_file(
+            local_path,
+            progress(type = "down", con = stderr())
+        )
+    tbl(conn, local_path) |>
+        filter(cell_ %in% cells)
+}
+
+#' Returns unharmonised metadata for a metadata query
+#' @inherit get_unharmonised_dataset description
+#' @param metadata A lazy data frame obtained from [get_metadata()], filtered
+#'  down to some cells of interest
+#' @inheritDotParams get_unharmonised_dataset
+#' @return A tibble with two columns:
+#'  * `file_id`: the same `file_id` as the main metadata table obtained from [get_metadata()]
+#'  * `unharmonised`: a nested tibble, with one row per cell in the input `metadata`, containing unharmonised metadata
+#' @export
+#' @importFrom dplyr group_by summarise filter collect
+#' @examples
+#' harmonised <- get_metadata() |> dplyr::filter(tissue == "kidney blood vessel")
+#' unharmonised <- get_unharmonised_metadata(harmonised)
+get_unharmonised_metadata = function(metadata, ...){
+    args = list(...)
+    metadata |>
+        collect() |>
+        group_by(file_id) |>
+        summarise(
+            unharmonised = list(dataset_id=file_id[[1]], cells=cell_, conn=metadata$src$con) |>
+                c(args) |> 
+                do.call(get_unharmonised_dataset, args=_) |> 
+                list()
+        )
+}