diff --git a/CHANGELOG.md b/CHANGELOG.md index a3c1834ddf7..730491e0ad0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -268,6 +268,9 @@ ### Build tool +- Dependency resolution will use cache when offline. + ([Ben Ownby](https://github.com/benev0)) + - New projects are generated using OTP28 on GitHub Actions. ([Louis Pilfold](https://github.com/lpil)) diff --git a/compiler-cli/src/dependencies.rs b/compiler-cli/src/dependencies.rs index 3b40dfd9ce7..28b7de987d3 100644 --- a/compiler-cli/src/dependencies.rs +++ b/compiler-cli/src/dependencies.rs @@ -3,6 +3,7 @@ mod dependency_manager; use std::{ cell::RefCell, collections::{HashMap, HashSet}, + io::Read, process::Command, rc::Rc, time::Instant, @@ -12,16 +13,20 @@ use camino::{Utf8Path, Utf8PathBuf}; use ecow::{EcoString, eco_format}; use flate2::read::GzDecoder; use gleam_core::{ - Error, Result, + Error, Result, Warning, build::{Mode, Target, Telemetry}, config::PackageConfig, dependency::{self, PackageFetchError}, error::{FileIoAction, FileKind, ShellCommandFailureReason, StandardIoAction}, hex::{self, HEXPM_PUBLIC_KEY}, - io::{HttpClient as _, TarUnpacker, WrappedReader}, + io::{FileSystemReader, FileSystemWriter, HttpClient as _, TarUnpacker, WrappedReader}, manifest::{Base16Checksum, Manifest, ManifestPackage, ManifestPackageSource, Resolved}, - paths::ProjectPaths, + paths::{ + ProjectPaths, global_hexpm_package_release_response_cache, + global_hexpm_packages_response_cache, + }, requirement::Requirement, + warning::WarningEmitterIO, }; use hexpm::version::Version; use itertools::Itertools; @@ -37,7 +42,7 @@ use crate::{ TreeOptions, build_lock::{BuildLock, Guard}, cli, - fs::{self, ProjectIO}, + fs::{self, ConsoleWarningEmitter, ProjectIO}, http::HttpClient, }; @@ -1093,8 +1098,52 @@ async fn lookup_package( Some(provided_package) => Ok(provided_package.to_manifest_package(name.as_str())), None => { let config = hexpm::Config::new(); - let release = - hex::get_package_release(&name, &version, &config, &HttpClient::new()).await?; + // performance may be able to be improved by reading from local cache first + // depending on the volatility of the endpoint + // retirement status volatile, content volatile one hour after initial publication + // content may be edited or deleted by admins + let mut resp_body = Vec::new(); + let release = hex::get_package_release( + &name, + &version, + &config, + Some(&mut resp_body), + &HttpClient::new(), + ) + .await; + let fs = ProjectIO::new(); + let cache_path = + global_hexpm_package_release_response_cache(&name, &version.to_string()); + let release = match release { + Ok(rel) => { + let _ = fs.write_bytes(&cache_path, &resp_body); + rel + } + Err(_) => { + tracing::debug!(name=%name, version=%version, "fetching_release_data_from_json_cache_file"); + let cached_result = + fs.read_bytes(&cache_path).map_err(|err| Error::FileIo { + action: FileIoAction::Read, + kind: FileKind::File, + path: cache_path.clone(), + err: Some(err.to_string()), + })?; + let release = + serde_json::from_slice(&cached_result).map_err(|err| Error::FileIo { + action: FileIoAction::Read, + kind: FileKind::File, + path: cache_path, + err: Some(err.to_string()), + })?; + + ConsoleWarningEmitter.emit_warning(Warning::LocalCache { + message: "json cache is not secure; verify dependencies when online." + .into(), + }); + + release + } + }; let build_tools = release .meta .build_tools @@ -1187,10 +1236,34 @@ impl dependency::PackageFetcher for PackageFetcher { let response = self .runtime .block_on(self.http.send(request)) - .map_err(PackageFetchError::fetch_error)?; + .map_err(PackageFetchError::fetch_error); + + let fs = ProjectIO::new(); + let cache_path = &global_hexpm_packages_response_cache(package); + let pkg = match response { + Ok(resp) => { + tracing::debug!(package = package, "saving_hex_package"); + let _ = fs.write_bytes(cache_path, resp.body()); + hexpm::repository_v2_get_package_response(resp, HEXPM_PUBLIC_KEY) + } + Err(_err) => { + tracing::debug!(package = package, "fetching_package_data_from_cache"); + let reader = fs + .reader(cache_path) + .map_err(PackageFetchError::fetch_error)?; + let mut decoder = GzDecoder::new(reader); + let mut data = Vec::new(); + let _ = decoder + .read_to_end(&mut data) + .map_err(PackageFetchError::fetch_error)?; + ConsoleWarningEmitter.emit_warning(Warning::LocalCache { + message: "Hexpm repository cache used; dependencies may be outdated.".into(), + }); + hexpm::repository_v2_package_parse_body(&data, HEXPM_PUBLIC_KEY) + } + } + .map_err(PackageFetchError::from)?; - let pkg = hexpm::repository_v2_get_package_response(response, HEXPM_PUBLIC_KEY) - .map_err(PackageFetchError::from)?; let pkg = Rc::new(pkg); let pkg_ref = Rc::clone(&pkg); self.cache_package(package, pkg); diff --git a/compiler-core/src/hex.rs b/compiler-core/src/hex.rs index 408ed96d6ae..72b732a4592 100644 --- a/compiler-core/src/hex.rs +++ b/compiler-core/src/hex.rs @@ -316,6 +316,7 @@ pub async fn get_package_release( name: &str, version: &Version, config: &hexpm::Config, + raw_response: Option<&mut Vec>, http: &Http, ) -> Result> { let version = version.to_string(); @@ -326,5 +327,8 @@ pub async fn get_package_release( ); let request = hexpm::api_get_package_release_request(name, &version, None, config); let response = http.send(request).await?; + if let Some(data) = raw_response { + data.clone_from(response.body()); + } hexpm::api_get_package_release_response(response).map_err(Error::hex) } diff --git a/compiler-core/src/paths.rs b/compiler-core/src/paths.rs index ff6079e00d9..90a4380178f 100644 --- a/compiler-core/src/paths.rs +++ b/compiler-core/src/paths.rs @@ -130,6 +130,22 @@ pub fn global_package_cache_package_tarball(package_name: &str, version: &str) - global_packages_cache().join(format!("{package_name}-{version}.tar")) } +pub fn global_hexpm_packages_response_cache(package_name: &str) -> Utf8PathBuf { + global_hexpm_response_cache_path().join(format!("packages-{package_name}.gz")) +} + +pub fn global_hexpm_package_release_response_cache( + package_name: &str, + version: &str, +) -> Utf8PathBuf { + global_hexpm_response_cache_path() + .join(format!("packages-{package_name}-releases-{version}.json")) +} + +fn global_hexpm_response_cache_path() -> Utf8PathBuf { + global_hexpm_cache().join("response") +} + pub fn global_hexpm_credentials_path() -> Utf8PathBuf { global_hexpm_cache().join("credentials") } diff --git a/compiler-core/src/warning.rs b/compiler-core/src/warning.rs index 37f2b403a71..a77419fa6fa 100644 --- a/compiler-core/src/warning.rs +++ b/compiler-core/src/warning.rs @@ -167,6 +167,10 @@ pub enum Warning { DeprecatedEnvironmentVariable { variable: DeprecatedEnvironmentVariable, }, + + LocalCache { + message: EcoString, + }, } #[derive(Debug, Clone, Eq, PartialEq, Copy)] @@ -1422,6 +1426,14 @@ The imported value could not be used in this module anyway." location: None, } } + + Warning::LocalCache { message } => Diagnostic { + title: "Use of cached files".into(), + text: message.into(), + level: diagnostic::Level::Warning, + location: None, + hint: None, + }, } }