accepting counts from multiple databases

Author: myushen

DESCRIPTION

@@ -121,7 +121,7 @@ biocViews:
     Transcription,
     Transcriptomics
 Encoding: UTF-8
-RoxygenNote: 7.2.3
+RoxygenNote: 7.3.1
 LazyDataCompression: xz
 URL: https://github.com/stemangiola/CuratedAtlasQueryR
 BugReports: https://github.com/stemangiola/CuratedAtlasQueryR/issues

NAMESPACE

@@ -1,9 +1,9 @@
 # Generated by roxygen2: do not edit by hand
 
 S3method(as.sparse,DelayedMatrix)
-export(get_database_url)
 export(SAMPLE_DATABASE_URL)
 export(get_SingleCellExperiment)
+export(get_database_url)
 export(get_metadata)
 export(get_seurat)
 export(get_single_cell_experiment)
@@ -62,7 +62,6 @@ importFrom(purrr,pmap_chr)
 importFrom(purrr,reduce)
 importFrom(purrr,set_names)
 importFrom(purrr,walk)
-importFrom(purrr,walk2)
 importFrom(rlang,.data)
 importFrom(stats,setNames)
 importFrom(stringr,str_remove_all)

R/metadata.R

@@ -6,32 +6,28 @@ NULL
 #' Environment that we use to cache the DuckDB connections
 #' @noRd
 cache <- rlang::env(
-    metadata_table = rlang::env()
+  metadata_table = rlang::env()
 )
 
 #' Returns the URLs for all metadata files 
+#' @param databases A character vector specifying the names of the metadata files. Default is c("metadata.0.2.3.parquet", "fibrosis.0.2.3.parquet")
 #' @export
-#' @return A named character vector whose names are parquet file names, and whose values are URLs
+#' @return A character vector of URLs to parquet files to download
 #' @examples
-#' get_metadata(remote_url = get_database_url("metadata.0.2.3.parquet"))
-
-
+#' get_database_url("metadata.0.2.3.parquet")
 get_database_url <- function(databases = c("metadata.0.2.3.parquet", "fibrosis.0.2.3.parquet")) {
-    glue::glue(
-      "https://object-store.rc.nectar.org.au/v1/AUTH_06d6e008e3e642da99d806ba3ea629c5/metadata/{databases}"
-    ) |>
-    setNames(databases)
+  glue::glue(
+    "https://object-store.rc.nectar.org.au/v1/AUTH_06d6e008e3e642da99d806ba3ea629c5/metadata/{databases}")
 }
 
- 
 #' URL pointing to the sample metadata file, which is smaller and for test,
 #' demonstration, and vignette purposes only
 #' @export
 #' @return A character scalar consisting of the URL
 #' @examples
-#' get_metadata(remote_url = SAMPLE_DATABASE_URL)
+#' get_metadata(remote_url = SAMPLE_DATABASE_URL, cache_directory = tempdir())
 SAMPLE_DATABASE_URL <- single_line_str(
-    "https://object-store.rc.nectar.org.au/v1/
+  "https://object-store.rc.nectar.org.au/v1/
     AUTH_06d6e008e3e642da99d806ba3ea629c5/metadata/
     sample_metadata.0.2.3.parquet"
 )
@@ -73,7 +69,7 @@ SAMPLE_DATABASE_URL <- single_line_str(
 #' @importFrom httr progress
 #' @importFrom cli cli_alert_info hash_sha256
 #' @importFrom glue glue
-#' @importFrom purrr walk2
+#' @importFrom purrr walk
 #'
 #' @details
 #'
@@ -147,35 +143,39 @@ SAMPLE_DATABASE_URL <- single_line_str(
 #'
 #' get_metadata(cache_directory = path.expand('~'))
 #' 
-
-
 get_metadata <- function(
     remote_url = get_database_url(),
     cache_directory = get_default_cache_dir(),
     use_cache = TRUE
 ) {
-    hash <- c(remote_url, cache_directory) |> paste0(collapse="") |>
-        hash_sha256()
-    cached_connection <- cache$metadata_table[[hash]]
-    if (!is.null(cached_connection) && isTRUE(use_cache)) {
-        cached_connection
-    }
-    else {
-      db_path <- file.path(cache_directory, remote_url |> basename())
-      walk2(remote_url, db_path, function(url, path) {
-        if (!file.exists(path)) {
-          report_file_sizes(url)
-          sync_remote_file(url,
-                           path,
-                           progress(type = "down", con = stderr()))
-        }
-      })
-        
-        table <- duckdb() |>
-            dbConnect(drv = _, read_only = TRUE) |>
-            read_parquet(db_path)
-        cache$metadata_table[[hash]] <- table
-        table
+  # Synchronize remote files
+  walk(remote_url, function(url) {
+    # Calculate the file path from the URL
+    path <- file.path(cache_directory, url |> basename())
+    if (!file.exists(path)) {
+      report_file_sizes(url)
+      sync_remote_file(url,
+                       path,
+                       progress(type = "down", con = stderr()))
     }
+  })
+  all_parquet <- file.path(cache_directory, dir(cache_directory, pattern = ".parquet$"))
+  # We try to avoid re-reading a set of parquet files 
+  # that is identical to a previous set by hashing the file list
+  hash <- all_parquet |> paste0(collapse="") |>
+    hash_sha256()
+  cached_connection <- cache$metadata_table[[hash]]
+  
+  if (!is.null(cached_connection) && isTRUE(use_cache)) {
+    # If the file list is identical, just re-use the database table
+    cached_connection
+  }
+  else {
+    table <- duckdb() |>
+      dbConnect(drv = _, read_only = TRUE) |>
+      read_parquet(path = all_parquet)
+    cache$metadata_table[[hash]] <- table
+    table
+  }
 }
 

man/DATABASE_URL.Rd

@@ -50,8 +50,9 @@ find_figure <- function(names){
         file.path("man", "figures", names) 
 }
 METADATA_URL = if (params$demo_metadata)
-        CuratedAtlasQueryR::SAMPLE_DATABASE_URL else
-        CuratedAtlasQueryR::get_database_url
+       CuratedAtlasQueryR::SAMPLE_DATABASE_URL else
+       CuratedAtlasQueryR::get_database_url
+
 ```
 
 <!-- badges: start -->
@@ -98,12 +99,20 @@ library(CuratedAtlasQueryR)
 
 ### Load the metadata
 
-```{r}
+```{r, eval=FALSE}
 # Note: in real applications you should use the default value of remote_url 
 metadata <- get_metadata(remote_url = METADATA_URL)
 metadata
 ```
 
+```{r, echo=FALSE}
+# Note: a custom cache is used here ONLY for R CHECK compliance purposes. Users will NOT need to specify a custom cache
+metadata <- get_metadata(remote_url = METADATA_URL, cache_directory = tempdir())
+metadata
+```
+ 
+
+
 The `metadata` variable can then be re-used for all subsequent queries.
 
 ### Explore the tissue