Fix EASIN functions and increase time out for downloading railways data

Author: elgabbas

R/DWF_EASIN_Down.R

@@ -41,15 +41,13 @@ EASIN_Down <- function(
     AllArgs = AllArgs, Type = "numeric",
     Args = c("Timeout", "NSearch", "SleepTime"))
 
-
   # # ..................................................................... ###
 
   # Environment variables ----
-  IASDT.R::CatTime("Environment variables")
 
   EnvVars2Read <- tibble::tribble(
     ~VarName, ~Value, ~CheckDir, ~CheckFile,
-    "EASIN_URL", "DP_R_EASIN_url", FALSE, FALSE,
+    "EASIN_URL", "DP_R_EASIN_data_url", FALSE, FALSE,
     "Path_EASIN", "DP_R_EASIN_interim", FALSE, FALSE)
   # Assign environment variables and check file and paths
   IASDT.R::AssignEnvVars(EnvFile = EnvFile, EnvVarDT = EnvVars2Read)
@@ -90,6 +88,7 @@ EASIN_Down <- function(
 
       Path_Part <- IASDT.R::Path(
         Path_EASIN, "FileParts", paste0(Obj_Out, ".RData"))
+
       if (IASDT.R::CheckData(Path_Part, warning = FALSE)) {
         next
       }
@@ -131,7 +130,6 @@ EASIN_Down <- function(
         }
       }
 
-
       if (inherits(ChunkDT, "data.frame")) {
         IASDT.R::SaveAs(InObj = ChunkDT, OutObj = Obj_Out, OutPath = Path_Part)
 

R/DWF_EASIN_Process.R

@@ -66,6 +66,30 @@
 #' @export
 #' @order 1
 
+
+
+
+setwd("U:/Workflow/")
+ExtractTaxa = TRUE
+ExtractData = TRUE
+NDownTries = 10L
+NCores = 6L
+SleepTime = 10L
+NSearch = 1000L
+EnvFile = ".env"
+DeleteChunks = TRUE
+StartYear = 1981L
+Plot = TRUE
+
+
+
+
+
+
+
+
+
+
 EASIN_Process <- function(
     ExtractTaxa = TRUE, ExtractData = TRUE, NDownTries = 10L, NCores = 6L,
     SleepTime = 10L, NSearch = 1000L, EnvFile = ".env", DeleteChunks = TRUE,
@@ -116,7 +140,6 @@ EASIN_Process <- function(
     "Path_Grid_Ref", "DP_R_Grid_raw", TRUE, FALSE,
     "Path_EASIN", "DP_R_EASIN_processed", FALSE, FALSE,
     "Path_EASIN_Interim", "DP_R_EASIN_interim", FALSE, FALSE,
-    "EASIN_URL", "DP_R_EASIN_url", FALSE, FALSE,
     "TaxaInfoFile", "DP_R_Taxa_info_rdata", FALSE, TRUE,
     "EASIN_Ref", "DP_R_Taxa_easin", FALSE, TRUE)
   # Assign environment variables and check file and paths
@@ -175,7 +198,10 @@ EASIN_Process <- function(
 
   IASDT.R::CatTime("Extract EASIN taxonomy list")
 
-  if (ExtractTaxa) {
+  Path_EASIN_Taxa <- IASDT.R::Path(Path_EASIN, "EASIN_Taxa.RData")
+  Taxa_Okay <- IASDT.R::CheckData(Path_EASIN_Taxa, warning = FALSE)
+
+  if (ExtractTaxa || isFALSE(Taxa_Okay)) {
     IASDT.R::CatTime("Download EASIN taxa", Level = 1)
 
     # Download EASIN taxa
@@ -209,8 +235,7 @@ EASIN_Process <- function(
           "Cenchrus setaceus", 5828232, "R03000",
           "Neltuma juliflora", 5358460, "R12278",
           "Persicaria perfoliata", 4033648, "R19287",
-          "Pueraria montana (Lour.) Merr. var. lobata", 2977636, "R12644")
-      )
+          "Pueraria montana (Lour.) Merr. var. lobata", 2977636, "R12644"))
 
     EASIN_Taxa <- EASIN_Taxa_Orig %>%
       # Merge with EASIN reference list of taxonomy standardization
@@ -237,10 +262,10 @@ EASIN_Process <- function(
     }
     ## Save EASIN taxa - RData ----
     IASDT.R::CatTime("Save EASIN taxa - RData", Level = 1)
-    save(EASIN_Taxa, file = IASDT.R::Path(Path_EASIN, "EASIN_Taxa.RData"))
+    save(EASIN_Taxa, file = Path_EASIN_Taxa)
   } else {
     IASDT.R::CatTime("Loading EASIN taxa list")
-    load(IASDT.R::Path(Path_EASIN, "EASIN_Taxa.RData"))
+    EASIN_Taxa <- IASDT.R::LoadAs(Path_EASIN_Taxa)
   }
 
   # # ..................................................................... ###
@@ -252,6 +277,7 @@ EASIN_Process <- function(
   IASDT.R::CatTime("Download EASIN data")
 
   if (ExtractData) {
+
     TimeStartData <- lubridate::now(tzone = "CET")
 
     ## Prepare working on parallel ----
@@ -272,6 +298,8 @@ EASIN_Process <- function(
       on.exit(future::plan("future::sequential", gc = TRUE), add = TRUE)
     }
 
+    IASDT.R::CatTime("Processing EASIN data", Level = 1)
+
     # Start downloading, allow for a maximum of `NumDownTries` trials
     Try <- 0
 
@@ -302,7 +330,6 @@ EASIN_Process <- function(
         break
       }
 
-      IASDT.R::CatTime("Processing EASIN data", Level = 1)
       Down <- try(
         future.apply::future_lapply(
           X = NotProcessed, FUN = IASDT.R::EASIN_Down, EnvFile = EnvFile,

R/DWF_EASIN_Taxonomy.R

@@ -28,11 +28,10 @@ EASIN_Taxonomy <- function(
   # # ..................................................................... ###
 
   # Environment variables ----
-  IASDT.R::CatTime("Environment variables")
 
   EnvVars2Read <- tibble::tribble(
     ~VarName, ~Value, ~CheckDir, ~CheckFile,
-    "EASIN_URL", "DP_R_EASIN_url", FALSE, FALSE)
+    "EASIN_URL", "DP_R_EASIN_taxa_url", FALSE, FALSE)
   # Assign environment variables and check file and paths
   IASDT.R::AssignEnvVars(EnvFile = EnvFile, EnvVarDT = EnvVars2Read)
   rm(EnvVars2Read, envir = environment())

R/DWF_Railway_Intensity.R

@@ -296,12 +296,12 @@ Railway_Intensity <- function(
           expr = {
 
             stringr::str_glue(
-              'curl -k -L --connect-timeout 120 --max-time 1800 --retry 5 \\
+              'curl -k -L --connect-timeout 180 --max-time 3600 --retry 5 \\
             "{URL}" -o "{Path}" --silent') %>%
               system()
 
             Success <- IASDT.R::CheckZip(Path)
-            
+
             if (isFALSE(Success)) {
               fs::file_delete(Path)
             }