allow duckdb to choose memory limit, drop memuse dependency (#167)

* drop memuse dependency, allow duckdb to detect avaialble ram by default

Author: e-kotov

DESCRIPTION

@@ -41,7 +41,6 @@ Imports:
     lifecycle,
     lubridate,
     memoise,
-    memuse,
     openssl,
     parallelly,
     paws.storage (>= 0.4.0),

NEWS.md

@@ -14,6 +14,8 @@
 
 * Metadata fetched by `spod_available_data()` has extra columns such as data `type`, `zones` and `period`, see help `?spod_available_data()` for details.
 
+* Memory allocation is now delegated to `DuckDB` engine, which by default uses 80% of available RAM. Beware that in some HPC environments this may detect more memory than is actually available to your job, so set the limit manually to 80% of RAM available to your job with `max_mem_gb` argument of `spod_get()`, `spod_convert()`, `spod_connect()` functions. This will also improve performance in some cases, as DuckDB is more efficient than R at memory allocation (PR [#167](https://github.com/rOpenSpain/spanishoddata/pull/167)).
+
 ## Bug fixes
 
 * More reliable, but still multi-threaded data file downloads using base R `utils::download.file()` instead of `curl::multi_download()` which failed on some connections (issue [#127](https://github.com/rOpenSpain/spanishoddata/issues/127)), so now `curl` dependency is no longer required. PR [#165](https://github.com/rOpenSpain/spanishoddata/pull/165).

R/connect.R

@@ -39,15 +39,15 @@ spod_connect <- function(
   data_path,
   target_table_name = NULL,
   quiet = FALSE,
-  max_mem_gb = max(4, spod_available_ram() - 4),
+  max_mem_gb = NULL,
   max_n_cpu = max(1, parallelly::availableCores() - 1),
   temp_path = spod_get_temp_dir()
 ) {
   # Validate imputs
   checkmate::assert_access(data_path, access = 'r')
   checkmate::assert_character(target_table_name, null.ok = TRUE)
   checkmate::assert_flag(quiet)
-  checkmate::assert_number(max_mem_gb, lower = 1)
+  checkmate::assert_number(max_mem_gb, lower = 1, null.ok = TRUE)
   checkmate::assert_integerish(max_n_cpu, lower = 1)
   checkmate::assert_directory_exists(temp_path, access = "rw")
 

R/convert.R

@@ -628,21 +628,23 @@ spod_sql_where_dates <- function(dates) {
 
 #' Set maximum memory and number of threads for a `DuckDB` connection
 #' @param con A `duckdb` connection
-#' @param max_mem_gb The maximum memory to use in GB. A conservative default is 3 GB, which should be enough for resaving the data to `DuckDB` form a folder of CSV.gz files while being small enough to fit in memory of most even old computers. For data analysis using the already converted data (in `DuckDB` or Parquet format) or with the raw CSV.gz data, it is recommended to increase it according to available resources.
+#' @param max_mem_gb `integer` value of the maximum operating memory to use in GB. `NULL` by default, delegates the choice to the `DuckDB` engine which usually sets it to 80% of available memory. Caution, in HPC use, the amount of memory available to your job may be determined incorrectly by the `DuckDB` engine, so it is recommended to set this parameter explicitly according to your job's memory limits.
 #' @param max_n_cpu The maximum number of threads to use. Defaults to the number of available cores minus 1.
 #' @return A `duckdb` connection.
 #' @keywords internal
 spod_duckdb_limit_resources <- function(
   con,
-  max_mem_gb = max(4, spod_available_ram() - 4),
+  max_mem_gb = NULL,
   max_n_cpu = max(1, parallelly::availableCores() - 1)
 ) {
-  DBI::dbExecute(
-    con,
-    dplyr::sql(
-      glue::glue("SET max_memory='{max_mem_gb}GB';")
+  if (!is.null(max_mem_gb)) {
+    DBI::dbExecute(
+      con,
+      dplyr::sql(
+        glue::glue("SET max_memory='{max_mem_gb}GB';")
+      )
     )
-  )
+  }
 
   DBI::dbExecute(
     con,

R/duckdb-helpers.R

@@ -63,7 +63,7 @@ spod_get <- function(
   dates = NULL,
   data_dir = spod_get_data_dir(),
   quiet = FALSE,
-  max_mem_gb = max(4, spod_available_ram() - 4),
+  max_mem_gb = NULL,
   max_n_cpu = max(1, parallelly::availableCores() - 1),
   max_download_size_gb = 1,
   duckdb_target = ":memory:",
@@ -100,7 +100,7 @@ spod_get <- function(
     )
   )
   checkmate::assert_flag(quiet)
-  checkmate::assert_number(max_mem_gb, lower = 1)
+  checkmate::assert_number(max_mem_gb, lower = 1, null.ok = TRUE)
   checkmate::assert_integerish(max_n_cpu, lower = 1)
   checkmate::assert_number(max_download_size_gb, lower = 0.1)
   checkmate::assert_string(duckdb_target)

R/get.R

@@ -394,18 +394,6 @@ spod_match_data_type <- function(
   return(NULL)
 }
 
-#' Get available RAM
-#' @keywords internal
-#' @return A `numeric` amount of available RAM in GB.
-spod_available_ram <- function() {
-  return(
-    as.numeric(unclass(memuse::Sys.meminfo())[1][['totalram']]) /
-      1024 /
-      1024 /
-      1024
-  )
-}
-
 #' Remove duplicate values in a semicolon-separated string
 #'
 #' @description