refactor: add code to retry download

Author: jorainer

.Rbuildignore

@@ -10,3 +10,5 @@ local_data
 NEWS.md
 README.Rmd
 logo.png
+pkgdown/*
+_pkgdown.yml

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: MsBackendMetaboLights
 Title: Retrieve Mass Spectrometry Data from MetaboLights
-Version: 1.1.1
+Version: 1.1.2
 Authors@R:
     c(person(given = "Johannes", family = "Rainer",
              email = "Johannes.Rainer@eurac.edu",
@@ -27,7 +27,8 @@ Imports:
     ProtGenerics,
     BiocFileCache,
     S4Vectors,
-    methods
+    methods,
+    progress
 Suggests:
     testthat,
     rmarkdown,

NAMESPACE

@@ -23,6 +23,8 @@ importFrom(methods,as)
 importFrom(methods,callNextMethod)
 importFrom(methods,new)
 importFrom(methods,validObject)
+importFrom(progress,progress_bar)
+importFrom(utils,capture.output)
 importFrom(utils,read.table)
 importMethodsFrom(BiocFileCache,"bfcmeta<-")
 importMethodsFrom(BiocFileCache,bfcmetalist)

NEWS.md

@@ -1,3 +1,11 @@
+# MsBackendMetaboLights 1.1
+
+## Changes in 1.1.2
+
+- Fetch and cache each data file individually.
+- Retry retrieval of data from MetaboLights up to 3 times before throwing an
+  error message, with an increasing time period between attempts.
+
 # MsBackendMetaboLights 1.0
 
 ## Changes in 1.0.1
@@ -29,4 +37,4 @@
 
 ## Changes in 0.0.1
 
-- Add utility functions to retrieve information from MetaboLights.
+- Add utility functions to retrieve information from MetaboLights.

R/MsBackendMetaboLights.R

@@ -45,6 +45,17 @@
 #' will check the local cache and eventually download missing data files from
 #' the MetaboLights repository.
 #'
+#' @note
+#'
+#' To account for high server load and eventually failing or rejected
+#' downloads from the MetaboLights ftp server, the download functions
+#' repeatedly retry to download a file. An error is thrown if download fails
+#' for 3 consecutive attempts. Between each attemp, the function waits
+#' for an increasing time period (5 seconds between the first and second
+#' and 10 seconds between the 2nd and 3rd attempt). This time period can
+#' also be configured with the `"metabolights.sleep_mult"` option, which
+#' defines the *sleep time multiplicator* (defaults to 5).
+#'
 #' @param object an instance of `MsBackendMetaboLights`.
 #'
 #' @param mtblsId `character(1)` with the ID of a single MetaboLights data
@@ -419,13 +430,15 @@ mtbls_list_files <- function(x = character(), pattern = NULL) {
     cu <- new_handle()
     handle_setopt(cu, ftp_use_epsv = TRUE, dirlistonly = TRUE)
     tryCatch({
-        con <- curl(url = mtbls_ftp_path(x, mustWork = FALSE), "r", handle = cu)
+        con <- .retry(
+            curl(url = mtbls_ftp_path(x, mustWork = FALSE), "r", handle = cu),
+            sleep_mult = .sleep_mult())
     }, error = function(e) {
         stop("Failed to connect to MetaboLights. No internet connection? ",
              "Does the data set \"", x, "\" exist?\n - ", e$message,
              call. = FALSE)
     })
-    fls <- readLines(con)
+    fls <- .retry(readLines(con), sleep_mult = .sleep_mult())
     close(con)
     if (length(pattern))
         fls[grepl(pattern, fls)]
@@ -446,9 +459,10 @@ mtbls_list_files <- function(x = character(), pattern = NULL) {
     fpath <- mtbls_ftp_path(x, mustWork = FALSE)
     a_fls <- mtbls_list_files(x, pattern = "^a_")
     res <- lapply(a_fls, function(z) {
-        read.table(paste0(fpath, z),
-                   sep = "\t", header = TRUE,
-                   check.names = FALSE)
+        .retry(read.table(paste0(fpath, z),
+                          sep = "\t", header = TRUE,
+                          check.names = FALSE),
+               sleep_mult = .sleep_mult())
     })
     names(res) <- a_fls
     res
@@ -534,10 +548,20 @@ mtbls_cached_data_files <- function(mtblsId = character(),
 #'   file/table
 #' - `"rpath"`: the name of the cached data file (full local path)
 #'
+#' @note
+#'
+#' Download from MsBackendMetaboLights is tried 3 times with an increasing time
+#' delay between tries that can be configured using the
+#' `"metabolights.sleep_mult"` option.
+#'
 #' @importFrom BiocFileCache BiocFileCache
 #'
+#' @importFrom progress progress_bar
+#'
 #' @importMethodsFrom BiocFileCache bfcrpath bfcmeta<-
 #'
+#' @importFrom utils capture.output
+#'
 #' @noRd
 .mtbls_data_files <- function(mtblsId = character(), assayName = character(),
                               pattern = "mzML$|CDF$|mzXML$",
@@ -576,7 +600,20 @@ mtbls_cached_data_files <- function(mtblsId = character(),
     }
     ## Cache files
     bfc <- BiocFileCache()
-    lfiles <- bfcrpath(bfc, paste0(fpath, ffiles), fname = "exact")
+    pb <- progress_bar$new(format = paste0("[:bar] :current/:",
+                                           "total (:percent) in ",
+                                           ":elapsed"),
+                           total = length(ffiles), clear = FALSE)
+    lfiles <- unlist(lapply(ffiles, function(z) {
+        pb$tick()
+        invisible(capture.output(suppressMessages(
+            f <- .retry(bfcrpath(bfc, paste0(fpath, z), fname = "exact",
+                                 progress = list()),
+                        sleep_mult = .sleep_mult()))))
+        f
+    }))
+
+    ## lfiles <- bfcrpath(bfc, paste0(fpath, ffiles), fname = "exact")
     ## Add and store metadata to the cached files
     mdata <- data.frame(
         rid = names(lfiles),
@@ -616,4 +653,4 @@ mtbls_cached_data_files <- function(mtblsId = character(),
              "parameters.", call. = FALSE)
     res[, c("rid", "mtbls_id", "mtbls_assay_name",
             "derived_spectral_data_file", "rpath")]
-}
+}

R/retry_download.R

@@ -0,0 +1,37 @@
+#' Retry an expression `ntimes` times waiting an increasing amount of time
+#' between tries, i.e. waiting for `Sys.sleep(i * sleep_mult)` seconds between
+#' each try. If `expr` fails for `ntimes` times the error will be thrown.
+#'
+#' @param expr Expression to be evaluated.
+#'
+#' @param ntimes `integer(1)` with the number of times to try.
+#'
+#' @param sleep_mult `numeric(1)` multiplier to define the increasing waiting
+#'     time.
+#'
+#' @note
+#'
+#' Warnings are suppressed.
+#'
+#' @author Johannes Rainer
+#'
+#' @importFrom methods is
+#'
+#' @noRd
+.retry <- function(expr, ntimes = 3, sleep_mult = 0) {
+    res <- NULL
+    for (i in seq_len(ntimes)) {
+        res <- suppressWarnings(tryCatch(expr, error = function(e) e))
+        if (is(res, "simpleError")) {
+            if (i == ntimes)
+                stop(res)
+            message("i: ", i)
+            Sys.sleep(i * sleep_mult)
+        } else break
+    }
+    res
+}
+
+.sleep_mult <- function() {
+    as.integer(getOption("metabolights.sleep_mult", default = 5))
+}