Offline package resolution

CHANGELOG.md

@@ -268,6 +268,9 @@
 
 ### Build tool
 
+- Dependency resolution will use cache when offline.
+  ([Ben Ownby](https://github.com/benev0))
+
 - New projects are generated using OTP28 on GitHub Actions.
   ([Louis Pilfold](https://github.com/lpil))
 

compiler-cli/src/dependencies.rs

@@ -3,6 +3,7 @@ mod dependency_manager;
 use std::{
     cell::RefCell,
     collections::{HashMap, HashSet},
+    io::Read,
     process::Command,
     rc::Rc,
     time::Instant,
@@ -18,9 +19,10 @@ use gleam_core::{
     dependency::{self, PackageFetchError},
     error::{FileIoAction, FileKind, ShellCommandFailureReason, StandardIoAction},
     hex::{self, HEXPM_PUBLIC_KEY},
-    io::{HttpClient as _, TarUnpacker, WrappedReader},
+    io::{FileSystemReader, FileSystemWriter, HttpClient as _, TarUnpacker, WrappedReader},
     manifest::{Base16Checksum, Manifest, ManifestPackage, ManifestPackageSource, Resolved},
     paths::ProjectPaths,
+    paths::{global_hexpm_package_release_response_cache, global_hexpm_packages_response_cache},
     requirement::Requirement,
 };
 use hexpm::version::Version;
@@ -1093,8 +1095,44 @@ async fn lookup_package(
         Some(provided_package) => Ok(provided_package.to_manifest_package(name.as_str())),
         None => {
             let config = hexpm::Config::new();
-            let release =
-                hex::get_package_release(&name, &version, &config, &HttpClient::new()).await?;
+            // performance may be able to be improved by reading from local cache first
+            // depending on the volatility of the endpoint
+            // retirement status volatile, content volatile one hour after initial publication
+            // content may be edited or deleted by admins
+            let mut resp_body = Vec::new();
+            let release = hex::get_package_release(
+                &name,
+                &version,
+                &config,
+                Some(&mut resp_body),
+                &HttpClient::new(),
+            )
+            .await;
+            let fs = ProjectIO::new();
+            let cache_path =
+                global_hexpm_package_release_response_cache(&name, &version.to_string());
+            let release = match release {
+                Ok(rel) => {
+                    let _ = fs.write_bytes(&cache_path, &resp_body);
+                    rel
+                }
+                Err(_) => {
+                    let cached_result =
+                        fs.read_bytes(&cache_path).map_err(|err| Error::FileIo {
+                            action: FileIoAction::Read,
+                            kind: FileKind::File,
+                            path: cache_path.clone(),
+                            err: Some(err.to_string()),
+                        })?;
+
+                    serde_json::from_slice(&cached_result).map_err(|err| Error::FileIo {
+                        action: FileIoAction::Read,
+                        kind: FileKind::File,
+                        path: cache_path,
+                        err: Some(err.to_string()),
+                    })?
+                }
+            };
             let build_tools = release
                 .meta
                 .build_tools
@@ -1187,10 +1225,31 @@ impl dependency::PackageFetcher for PackageFetcher {
         let response = self
             .runtime
             .block_on(self.http.send(request))
-            .map_err(PackageFetchError::fetch_error)?;
+            .map_err(PackageFetchError::fetch_error);
+
+        let fs = ProjectIO::new();
+        let cache_path = &global_hexpm_packages_response_cache(package);
+        let pkg = match response {
+            Ok(resp) => {
+                tracing::debug!(package = package, "saving_hex_package");
+                let _ = fs.write_bytes(cache_path, resp.body());
+                hexpm::repository_v2_get_package_response(resp, HEXPM_PUBLIC_KEY)
+            }
+            Err(_err) => {
+                tracing::debug!(package = package, "fetching_package_data_from_cache");
+                let reader = fs
+                    .reader(cache_path)
+                    .map_err(PackageFetchError::fetch_error)?;
+                let mut decoder = GzDecoder::new(reader);
+                let mut data = Vec::new();
+                let _ = decoder
+                    .read_to_end(&mut data)
+                    .map_err(PackageFetchError::fetch_error)?;
+                hexpm::repository_v2_package_parse_body(&data, HEXPM_PUBLIC_KEY)
+            }
+        }
+        .map_err(PackageFetchError::from)?;
 
-        let pkg = hexpm::repository_v2_get_package_response(response, HEXPM_PUBLIC_KEY)
-            .map_err(PackageFetchError::from)?;
         let pkg = Rc::new(pkg);
         let pkg_ref = Rc::clone(&pkg);
         self.cache_package(package, pkg);

compiler-core/src/hex.rs

@@ -316,6 +316,7 @@ pub async fn get_package_release<Http: HttpClient>(
     name: &str,
     version: &Version,
     config: &hexpm::Config,
+    raw_response: Option<&mut Vec<u8>>,
     http: &Http,
 ) -> Result<hexpm::Release<hexpm::ReleaseMeta>> {
     let version = version.to_string();
@@ -326,5 +327,8 @@ pub async fn get_package_release<Http: HttpClient>(
     );
     let request = hexpm::api_get_package_release_request(name, &version, None, config);
     let response = http.send(request).await?;
+    if let Some(data) = raw_response {
+        data.clone_from(response.body());
+    }
     hexpm::api_get_package_release_response(response).map_err(Error::hex)
 }

compiler-core/src/paths.rs

@@ -130,6 +130,22 @@ pub fn global_package_cache_package_tarball(package_name: &str, version: &str) -
     global_packages_cache().join(format!("{package_name}-{version}.tar"))
 }
 
+pub fn global_hexpm_packages_response_cache(package_name: &str) -> Utf8PathBuf {
+    global_hexpm_response_cache_path().join(format!("packages-{package_name}.gz"))
+}
+
+pub fn global_hexpm_package_release_response_cache(
+    package_name: &str,
+    version: &str,
+) -> Utf8PathBuf {
+    global_hexpm_response_cache_path()
+        .join(format!("packages-{package_name}-releases-{version}.json"))
+}
+
+fn global_hexpm_response_cache_path() -> Utf8PathBuf {
+    global_hexpm_cache().join("response")
+}
+
 pub fn global_hexpm_credentials_path() -> Utf8PathBuf {
     global_hexpm_cache().join("credentials")
 }