Merge pull request #57 from devrc-hub/feature/cache-include

feature: Add cache for remote loaded files

Author: Lispython

Cargo.lock

@@ -53,6 +53,7 @@ devrc-plugins = { path = "../plugins", version = "0.5.4"}
 devrc-core = { path = "../core", version = "0.5.4"}
 netrc-rs = "0.1.2"
 base64 = "0.21.2"
+duration-str = "0.5.1"
 
 [build-dependencies]
 datetime = { version = "0.5.2", default_features = false }

cli/Cargo.toml

@@ -0,0 +1,65 @@
+use std::{
+    fs::{self, File},
+    io::Write,
+    path::{Path, PathBuf},
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
+
+use sha256::digest;
+use url::Url;
+
+use crate::{errors::DevrcResult, loader::LoadingConfig};
+
+const DEVRC_CACHE_DIR_NAME: &str = "devrc";
+
+/// Get absolute path to devrc cache dir
+pub fn get_cache_path() -> Option<PathBuf> {
+    dirs_next::cache_dir().map(|path| Path::new(&path).join(DEVRC_CACHE_DIR_NAME))
+}
+
+pub fn get_file_cache_meta(url: &Url) -> Option<PathBuf> {
+    let hash = digest(url.as_str());
+    get_cache_path().map(|path| path.join(format!("{:}.cache", hash)))
+}
+
+#[derive(Debug, Default)]
+pub struct Cache {}
+
+pub fn save(url: &Url, content: &str) -> DevrcResult<()> {
+    if let Some(file) = get_file_cache_meta(url) {
+        if let Some(dir) = file.parent() {
+            if !dir.exists() {
+                fs::create_dir_all(dir)?;
+            }
+        }
+
+        let mut f = File::create(file)?;
+        f.write_all(content.as_bytes())?;
+    }
+    Ok(())
+}
+
+pub fn load(
+    url: &Url,
+    _loading_config: &LoadingConfig,
+    _checksum: Option<&str>,
+    ttl: &Duration,
+) -> Option<String> {
+    if let Some(file) = get_file_cache_meta(url) {
+        if !file.exists() {
+            return None;
+        }
+
+        if let Ok(modified) = fs::metadata(&file).ok()?.modified() {
+            let now_timestamp = SystemTime::now().duration_since(UNIX_EPOCH).ok()?;
+            let created_timestamp = modified.duration_since(UNIX_EPOCH).ok()?;
+
+            let duration = now_timestamp - created_timestamp;
+
+            if duration < *ttl {
+                return fs::read_to_string(file).ok();
+            }
+        }
+    }
+    None
+}

cli/src/cache.rs

@@ -1,5 +1,5 @@
 use crate::interpreter::InterpreterKind;
-use std::{env, fmt::Debug, path::PathBuf};
+use std::{env, fmt::Debug, path::PathBuf, time::Duration};
 
 use devrc_core::logging::LogLevel;
 
@@ -11,6 +11,7 @@ pub struct Config {
     pub dry_run: bool,
     pub default: Vec<String>,
     pub plugins: indexmap::IndexMap<String, PathBuf>,
+    pub cache_ttl: Option<Duration>,
 }
 
 impl Default for Config {
@@ -22,6 +23,7 @@ impl Default for Config {
             log_level: LogLevel::Info,
             default: vec![],
             plugins: indexmap::IndexMap::new(),
+            cache_ttl: None,
         }
     }
 }

cli/src/config.rs

@@ -118,6 +118,10 @@ impl Devrcfile {
                     self.config.plugins.insert(name, path);
                 }
             }
+
+            if let Some(duration) = config.cache_ttl {
+                self.config.cache_ttl = Some(duration)
+            }
         }
 
         Ok(())

cli/src/devrcfile.rs

@@ -40,6 +40,9 @@ pub struct UrlImport {
     pub ignore_errors: bool,
 
     pub checksum: String,
+
+    #[serde(default)]
+    pub headers: indexmap::IndexMap<String, String>,
 }
 
 #[derive(Debug, Deserialize, Clone, Default)]
@@ -129,6 +132,16 @@ impl LocalFileImport {
         match loading_location {
             Location::LocalFile(path) => fs::read_to_string(path).map_err(DevrcError::IoError),
             Location::Remote { url, auth } => {
+                if let Some(cache_ttl) = config.cache_ttl {
+                    if let Some(content) = crate::cache::load(&url, &config, None, &cache_ttl) {
+                        config.log_level.debug(
+                            &format!("\n==> Loading ENV URL CACHE: `{}` ...", &url),
+                            &config.designer.banner(),
+                        );
+                        return Ok(content);
+                    }
+                }
+
                 let client = reqwest::blocking::Client::new();
                 let mut headers_map: HeaderMap = HeaderMap::new();
 
@@ -261,6 +274,18 @@ impl UrlImport {
         let parsed_url =
             Url::parse(&self.url).map_err(|_| DevrcError::InvalidIncludeUrl(self.url.clone()))?;
 
+        if let Some(cache_ttl) = config.cache_ttl {
+            if let Some(content) =
+                crate::cache::load(&parsed_url, &config, Some(&self.checksum), &cache_ttl)
+            {
+                config.log_level.debug(
+                    &format!("\n==> Loading ENV URL CACHE: `{}` ...", &parsed_url),
+                    &config.designer.banner(),
+                );
+                return Ok(content);
+            }
+        }
+
         config.log_level.debug(
             &format!("\n==> Loading ENV FILE: `{:}` ...", &parsed_url),
             &config.designer.banner(),
@@ -280,6 +305,10 @@ impl UrlImport {
                     });
                 }
 
+                if config.cache_ttl.is_some() {
+                    crate::cache::save(&parsed_url, &content)?;
+                }
+
                 Ok(content)
             }
             Ok(response) => {

cli/src/env_file.rs

@@ -18,6 +18,7 @@
 extern crate log;
 
 pub mod auth;
+pub mod cache;
 pub mod cli;
 pub mod common;
 pub mod config;

cli/src/lib.rs

@@ -1,3 +1,5 @@
+use std::time::Duration;
+
 use devrc_core::{logging::LogLevel, workshop::Designer};
 
 #[derive(Debug, Clone)]
@@ -8,6 +10,8 @@ pub struct LoadingConfig {
     pub(crate) log_level: LogLevel,
 
     pub designer: Designer,
+
+    pub cache_ttl: Option<Duration>,
 }
 
 impl Default for LoadingConfig {
@@ -16,6 +20,7 @@ impl Default for LoadingConfig {
             level: 1,
             log_level: Default::default(),
             designer: Designer::default(),
+            cache_ttl: None,
         }
     }
 }
@@ -26,6 +31,7 @@ impl LoadingConfig {
             level: 1,
             log_level,
             designer: Designer::default(),
+            cache_ttl: None,
         }
     }
 
@@ -42,4 +48,10 @@ impl LoadingConfig {
             ..self
         }
     }
+    pub fn with_cache_ttl(self, ttl: Option<Duration>) -> Self {
+        Self {
+            cache_ttl: ttl,
+            ..self
+        }
+    }
 }

cli/src/loader.rs

@@ -1,5 +1,6 @@
 use crate::{de::deserialize_some, interpreter::InterpreterKind};
-use std::{fmt::Debug, path::PathBuf};
+use duration_str::deserialize_option_duration;
+use std::{fmt::Debug, path::PathBuf, time::Duration};
 
 use devrc_core::logging;
 use serde::Deserialize;
@@ -34,6 +35,9 @@ pub struct RawConfig {
 
     #[serde(default, deserialize_with = "deserialize_some")]
     pub plugins: Option<indexmap::IndexMap<String, PathBuf>>,
+
+    #[serde(default, deserialize_with = "deserialize_option_duration")]
+    pub cache_ttl: Option<Duration>,
 }
 
 #[derive(Debug, Deserialize, Clone, PartialEq, Eq, PartialOrd, Ord, Default)]