Offline package resolution

CHANGELOG.md

@@ -268,6 +268,9 @@
 
 ### Build tool
 
+- Dependency resolution will use cache when offline.
+  ([Ben Ownby](https://github.com/benev0))
+
 - New projects are generated using OTP28 on GitHub Actions.
   ([Louis Pilfold](https://github.com/lpil))
 

compiler-cli/src/dependencies.rs

@@ -3,6 +3,7 @@ mod dependency_manager;
 use std::{
     cell::RefCell,
     collections::{HashMap, HashSet},
+    io::Read,
     process::Command,
     rc::Rc,
     time::Instant,
@@ -12,16 +13,20 @@ use camino::{Utf8Path, Utf8PathBuf};
 use ecow::{EcoString, eco_format};
 use flate2::read::GzDecoder;
 use gleam_core::{
-    Error, Result,
+    Error, Result, Warning,
     build::{Mode, Target, Telemetry},
     config::PackageConfig,
     dependency::{self, PackageFetchError},
     error::{FileIoAction, FileKind, ShellCommandFailureReason, StandardIoAction},
     hex::{self, HEXPM_PUBLIC_KEY},
-    io::{HttpClient as _, TarUnpacker, WrappedReader},
+    io::{FileSystemReader, FileSystemWriter, HttpClient as _, TarUnpacker, WrappedReader},
     manifest::{Base16Checksum, Manifest, ManifestPackage, ManifestPackageSource, Resolved},
-    paths::ProjectPaths,
+    paths::{
+        ProjectPaths, global_hexpm_package_release_response_cache,
+        global_hexpm_packages_response_cache,
+    },
     requirement::Requirement,
+    warning::WarningEmitterIO,
 };
 use hexpm::version::Version;
 use itertools::Itertools;
@@ -37,7 +42,7 @@ use crate::{
     TreeOptions,
     build_lock::{BuildLock, Guard},
     cli,
-    fs::{self, ProjectIO},
+    fs::{self, ConsoleWarningEmitter, ProjectIO},
     http::HttpClient,
 };
 
@@ -1093,8 +1098,52 @@ async fn lookup_package(
         Some(provided_package) => Ok(provided_package.to_manifest_package(name.as_str())),
         None => {
             let config = hexpm::Config::new();
-            let release =
-                hex::get_package_release(&name, &version, &config, &HttpClient::new()).await?;
+            // performance may be able to be improved by reading from local cache first
+            // depending on the volatility of the endpoint
+            // retirement status volatile, content volatile one hour after initial publication
+            // content may be edited or deleted by admins
+            let mut resp_body = Vec::new();
+            let release = hex::get_package_release(
+                &name,
+                &version,
+                &config,
+                Some(&mut resp_body),
+                &HttpClient::new(),
+            )
+            .await;
+            let fs = ProjectIO::new();
+            let cache_path =
+                global_hexpm_package_release_response_cache(&name, &version.to_string());
+            let release = match release {
+                Ok(rel) => {
+                    let _ = fs.write_bytes(&cache_path, &resp_body);
+                    rel
+                }
+                Err(_) => {
+                    tracing::debug!(name=%name, version=%version, "fetching_release_data_from_json_cache_file");
+                    let cached_result =
+                        fs.read_bytes(&cache_path).map_err(|err| Error::FileIo {
+                            action: FileIoAction::Read,
+                            kind: FileKind::File,
+                            path: cache_path.clone(),
+                            err: Some(err.to_string()),
+                        })?;
+                    let release =
+                        serde_json::from_slice(&cached_result).map_err(|err| Error::FileIo {
+                            action: FileIoAction::Read,
+                            kind: FileKind::File,
+                            path: cache_path,
+                            err: Some(err.to_string()),
+                        })?;
+
+                    ConsoleWarningEmitter.emit_warning(Warning::LocalCache {
+                        message: "json cache is not secure; verify dependencies when online."
+                            .into(),
+                    });
+
+                    release
+                }
+            };
             let build_tools = release
                 .meta
                 .build_tools
@@ -1187,10 +1236,34 @@ impl dependency::PackageFetcher for PackageFetcher {
         let response = self
             .runtime
             .block_on(self.http.send(request))
-            .map_err(PackageFetchError::fetch_error)?;
+            .map_err(PackageFetchError::fetch_error);
+
+        let fs = ProjectIO::new();
+        let cache_path = &global_hexpm_packages_response_cache(package);
+        let pkg = match response {
+            Ok(resp) => {
+                tracing::debug!(package = package, "saving_hex_package");
+                let _ = fs.write_bytes(cache_path, resp.body());
+                hexpm::repository_v2_get_package_response(resp, HEXPM_PUBLIC_KEY)
+            }
+            Err(_err) => {
+                tracing::debug!(package = package, "fetching_package_data_from_cache");
+                let reader = fs
+                    .reader(cache_path)
+                    .map_err(PackageFetchError::fetch_error)?;
+                let mut decoder = GzDecoder::new(reader);
+                let mut data = Vec::new();
+                let _ = decoder
+                    .read_to_end(&mut data)
+                    .map_err(PackageFetchError::fetch_error)?;
+                ConsoleWarningEmitter.emit_warning(Warning::LocalCache {
+                    message: "Hexpm repository cache used; dependencies may be outdated.".into(),
+                });
+                hexpm::repository_v2_package_parse_body(&data, HEXPM_PUBLIC_KEY)
+            }
+        }
+        .map_err(PackageFetchError::from)?;
 
-        let pkg = hexpm::repository_v2_get_package_response(response, HEXPM_PUBLIC_KEY)
-            .map_err(PackageFetchError::from)?;
         let pkg = Rc::new(pkg);
         let pkg_ref = Rc::clone(&pkg);
         self.cache_package(package, pkg);

compiler-core/src/hex.rs

@@ -316,6 +316,7 @@ pub async fn get_package_release<Http: HttpClient>(
     name: &str,
     version: &Version,
     config: &hexpm::Config,
+    raw_response: Option<&mut Vec<u8>>,
     http: &Http,
 ) -> Result<hexpm::Release<hexpm::ReleaseMeta>> {
     let version = version.to_string();
@@ -326,5 +327,8 @@ pub async fn get_package_release<Http: HttpClient>(
     );
     let request = hexpm::api_get_package_release_request(name, &version, None, config);
     let response = http.send(request).await?;
+    if let Some(data) = raw_response {
+        data.clone_from(response.body());
+    }
     hexpm::api_get_package_release_response(response).map_err(Error::hex)
 }

compiler-core/src/paths.rs

@@ -130,6 +130,22 @@ pub fn global_package_cache_package_tarball(package_name: &str, version: &str) -
     global_packages_cache().join(format!("{package_name}-{version}.tar"))
 }
 
+pub fn global_hexpm_packages_response_cache(package_name: &str) -> Utf8PathBuf {
+    global_hexpm_response_cache_path().join(format!("packages-{package_name}.gz"))
+}
+
+pub fn global_hexpm_package_release_response_cache(
+    package_name: &str,
+    version: &str,
+) -> Utf8PathBuf {
+    global_hexpm_response_cache_path()
+        .join(format!("packages-{package_name}-releases-{version}.json"))
+}
+
+fn global_hexpm_response_cache_path() -> Utf8PathBuf {
+    global_hexpm_cache().join("response")
+}
+
 pub fn global_hexpm_credentials_path() -> Utf8PathBuf {
     global_hexpm_cache().join("credentials")
 }

compiler-core/src/warning.rs

@@ -167,6 +167,10 @@ pub enum Warning {
     DeprecatedEnvironmentVariable {
         variable: DeprecatedEnvironmentVariable,
     },
+
+    LocalCache {
+        message: EcoString,
+    },
 }
 
 #[derive(Debug, Clone, Eq, PartialEq, Copy)]
@@ -1422,6 +1426,14 @@ The imported value could not be used in this module anyway."
                     location: None,
                 }
             }
+
+            Warning::LocalCache { message } => Diagnostic {
+                title: "Use of cached files".into(),
+                text: message.into(),
+                level: diagnostic::Level::Warning,
+                location: None,
+                hint: None,
+            },
         }
     }