Merge pull request #1193 from mikrostew/fetch-yarn3

Fetch and resolve Yarn3 using @yarnpkg/cli-dist

Author: mikrostew

crates/volta-core/src/error/kind.rs

@@ -402,6 +402,11 @@ pub enum ErrorKind {
     /// Thrown when the shim binary is called directly, not through a symlink
     RunShimDirectly,
 
+    /// Thrown when there was an error setting a tool to executable
+    SetToolExecutable {
+        tool: String,
+    },
+
     /// Thrown when there was an error copying an unpacked tool to the image directory
     SetupToolImageError {
         tool: String,
@@ -494,6 +499,9 @@ pub enum ErrorKind {
     #[cfg(windows)]
     WriteUserPathError,
 
+    /// Thrown when a user attempts to install a version of Yarn2
+    Yarn2NotSupported,
+
     /// Thrown when there is an error fetching the latest version of Yarn
     YarnLatestFetchError {
         from_url: String,
@@ -1207,6 +1215,13 @@ Please verify your internet connection.",
 
 Please use the existing shims provided by Volta (node, yarn, etc.) to run tools."
             ),
+            ErrorKind::SetToolExecutable { tool } => write!(
+                f,
+                r#"Could not set "{}" to executable
+
+{}"#,
+                tool, PERMISSIONS_CTA
+            ),
             ErrorKind::SetupToolImageError { tool, version, dir } => write!(
                 f,
                 "Could not create environment for {} v{}
@@ -1361,6 +1376,12 @@ to {}
                 "Could not write Path environment variable.
 
 Please ensure you have permissions to edit your environment variables."
+            ),
+            ErrorKind::Yarn2NotSupported => write!(
+                f,
+                "Yarn version 2 is not recommended for use, and not supported by Volta.
+
+Please use version 3 or greater instead."
             ),
             ErrorKind::YarnLatestFetchError { from_url } => write!(
                 f,
@@ -1474,6 +1495,7 @@ impl ErrorKind {
             ErrorKind::RegistryFetchError { .. } => ExitCode::NetworkError,
             ErrorKind::RunShimDirectly => ExitCode::InvalidArguments,
             ErrorKind::SetupToolImageError { .. } => ExitCode::FileSystemError,
+            ErrorKind::SetToolExecutable { .. } => ExitCode::FileSystemError,
             ErrorKind::ShimCreateError { .. } => ExitCode::FileSystemError,
             ErrorKind::ShimRemoveError { .. } => ExitCode::FileSystemError,
             ErrorKind::StringifyBinConfigError => ExitCode::UnknownError,
@@ -1493,6 +1515,7 @@ impl ErrorKind {
             ErrorKind::WritePlatformError { .. } => ExitCode::FileSystemError,
             #[cfg(windows)]
             ErrorKind::WriteUserPathError => ExitCode::EnvironmentError,
+            ErrorKind::Yarn2NotSupported => ExitCode::NoVersionMatch,
             ErrorKind::YarnLatestFetchError { .. } => ExitCode::NetworkError,
             ErrorKind::YarnVersionNotFound { .. } => ExitCode::NoVersionMatch,
         }

crates/volta-core/src/tool/npm/resolve.rs

@@ -1,18 +1,13 @@
 //! Provides resolution of npm Version requirements into specific versions
 
 use super::super::registry::{
-    public_registry_index, PackageDetails, PackageIndex, RawPackageMetadata,
-    NPM_ABBREVIATED_ACCEPT_HEADER,
+    fetch_public_index, public_registry_index, PackageDetails, PackageIndex,
 };
-use super::super::registry_fetch_error;
-use crate::error::{Context, ErrorKind, Fallible};
+use crate::error::{ErrorKind, Fallible};
 use crate::hook::ToolHooks;
 use crate::session::Session;
-use crate::style::progress_spinner;
 use crate::tool::Npm;
 use crate::version::{VersionSpec, VersionTag};
-use attohttpc::header::ACCEPT;
-use attohttpc::Response;
 use log::debug;
 use semver::{Version, VersionReq};
 
@@ -41,16 +36,7 @@ fn fetch_npm_index(hooks: Option<&ToolHooks<Npm>>) -> Fallible<(String, PackageI
         _ => public_registry_index("npm"),
     };
 
-    let spinner = progress_spinner(format!("Fetching public registry: {}", url));
-    let metadata: RawPackageMetadata = attohttpc::get(&url)
-        .header(ACCEPT, NPM_ABBREVIATED_ACCEPT_HEADER)
-        .send()
-        .and_then(Response::error_for_status)
-        .and_then(Response::json)
-        .with_context(registry_fetch_error("npm", &url))?;
-
-    spinner.finish_and_clear();
-    Ok((url, metadata.into()))
+    fetch_public_index(url, "npm")
 }
 
 fn resolve_tag(tag: &str, hooks: Option<&ToolHooks<Npm>>) -> Fallible<Version> {

crates/volta-core/src/tool/registry.rs

@@ -1,9 +1,13 @@
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 
+use super::registry_fetch_error;
 use crate::error::{Context, ErrorKind, Fallible};
 use crate::fs::read_dir_eager;
+use crate::style::progress_spinner;
 use crate::version::{hashmap_version_serde, version_serde};
+use attohttpc::header::ACCEPT;
+use attohttpc::Response;
 use cfg_if::cfg_if;
 use semver::Version;
 use serde::Deserialize;
@@ -30,6 +34,19 @@ cfg_if! {
     }
 }
 
+pub fn fetch_public_index(url: String, name: &str) -> Fallible<(String, PackageIndex)> {
+    let spinner = progress_spinner(format!("Fetching public registry: {}", url));
+    let metadata: RawPackageMetadata = attohttpc::get(&url)
+        .header(ACCEPT, NPM_ABBREVIATED_ACCEPT_HEADER)
+        .send()
+        .and_then(Response::error_for_status)
+        .and_then(Response::json)
+        .with_context(registry_fetch_error(name, &url))?;
+
+    spinner.finish_and_clear();
+    Ok((url, metadata.into()))
+}
+
 pub fn public_registry_package(package: &str, version: &str) -> String {
     format!(
         "{}/-/{}-{}.tgz",
@@ -39,6 +56,18 @@ pub fn public_registry_package(package: &str, version: &str) -> String {
     )
 }
 
+// need package and filename for namespaced tools like @yarnpkg/cli-dist, which is located at
+//   https://registry.npmjs.org/@yarnpkg/cli-dist/-/cli-dist-1.2.3.tgz
+pub fn scoped_public_registry_package(scope: &str, package: &str, version: &str) -> String {
+    format!(
+        "{}/{}/-/{}-{}.tgz",
+        public_registry_index(scope),
+        package,
+        package,
+        version
+    )
+}
+
 /// Figure out the unpacked package directory name dynamically
 ///
 /// Packages typically extract to a "package" directory, but not always

crates/volta-core/src/tool/yarn/fetch.rs

@@ -1,12 +1,15 @@
 //! Provides fetcher for Yarn distributions
 
+use std::env;
 use std::fs::File;
 use std::path::Path;
 
 use super::super::download_tool_error;
-use super::super::registry::{find_unpack_dir, public_registry_package};
+use super::super::registry::{
+    find_unpack_dir, public_registry_package, scoped_public_registry_package,
+};
 use crate::error::{Context, ErrorKind, Fallible};
-use crate::fs::{create_staging_dir, create_staging_file, rename};
+use crate::fs::{create_staging_dir, create_staging_file, rename, set_executable};
 use crate::hook::ToolHooks;
 use crate::layout::volta_home;
 use crate::style::{progress_bar, tool_version};
@@ -79,11 +82,14 @@ fn unpack_archive(archive: Box<dyn Archive>, version: &Version) -> Fallible<()>
             version: version_string.clone(),
         })?;
 
+    let unpack_dir = find_unpack_dir(temp.path())?;
+    // "bin/yarn" is not executable in the @yarnpkg/cli-dist package
+    ensure_bin_is_executable(&unpack_dir, "yarn")?;
+
     let dest = volta_home()?.yarn_image_dir(&version_string);
     ensure_containing_dir_exists(&dest)
         .with_context(|| ErrorKind::ContainingDirError { path: dest.clone() })?;
 
-    let unpack_dir = find_unpack_dir(temp.path())?;
     rename(unpack_dir, &dest).with_context(|| ErrorKind::SetupToolImageError {
         tool: "Yarn".into(),
         version: version_string.clone(),
@@ -122,7 +128,17 @@ fn determine_remote_url(version: &Version, hooks: Option<&ToolHooks<Yarn>>) -> F
             let distro_file_name = Yarn::archive_filename(&version_str);
             hook.resolve(version, &distro_file_name)
         }
-        _ => Ok(public_registry_package("yarn", &version_str)),
+        _ => {
+            if env::var_os("VOLTA_FEATURE_YARN_3").is_some() && version.major >= 2 {
+                Ok(scoped_public_registry_package(
+                    "@yarnpkg",
+                    "cli-dist",
+                    &version_str,
+                ))
+            } else {
+                Ok(public_registry_package("yarn", &version_str))
+            }
+        }
     }
 }
 
@@ -138,3 +154,8 @@ fn fetch_remote_distro(
         url,
     ))
 }
+
+fn ensure_bin_is_executable(unpack_dir: &Path, tool: &str) -> Fallible<()> {
+    let exec_path = unpack_dir.join("bin").join(tool);
+    set_executable(&exec_path).with_context(|| ErrorKind::SetToolExecutable { tool: tool.into() })
+}

crates/volta-core/src/tool/yarn/resolve.rs

@@ -1,8 +1,9 @@
 //! Provides resolution of Yarn requirements into specific versions
 
+use std::env;
+
 use super::super::registry::{
-    public_registry_index, PackageDetails, PackageIndex, RawPackageMetadata,
-    NPM_ABBREVIATED_ACCEPT_HEADER,
+    fetch_public_index, public_registry_index, PackageDetails, PackageIndex,
 };
 use super::super::registry_fetch_error;
 use super::metadata::{RawYarnIndex, YarnIndex};
@@ -12,7 +13,6 @@ use crate::session::Session;
 use crate::style::progress_spinner;
 use crate::tool::Yarn;
 use crate::version::{parse_version, VersionSpec, VersionTag};
-use attohttpc::header::ACCEPT;
 use attohttpc::Response;
 use log::debug;
 use semver::{Version, VersionReq};
@@ -69,23 +69,30 @@ fn resolve_semver(matching: VersionReq, hooks: Option<&ToolHooks<Yarn>>) -> Fall
     }
 }
 
-fn fetch_yarn_index() -> Fallible<(String, PackageIndex)> {
-    let url = public_registry_index("yarn");
-    let spinner = progress_spinner(format!("Fetching public registry: {}", url));
-    let metadata: RawPackageMetadata = attohttpc::get(&url)
-        .header(ACCEPT, NPM_ABBREVIATED_ACCEPT_HEADER)
-        .send()
-        .and_then(Response::error_for_status)
-        .and_then(Response::json)
-        .with_context(registry_fetch_error("Yarn", &url))?;
-
-    spinner.finish_and_clear();
-    Ok((url, metadata.into()))
+fn fetch_yarn_index(package: &str) -> Fallible<(String, PackageIndex)> {
+    let url = public_registry_index(package);
+    fetch_public_index(url, "Yarn")
 }
 
 fn resolve_custom_tag(tag: String) -> Fallible<Version> {
-    let (url, mut index) = fetch_yarn_index()?;
+    if env::var_os("VOLTA_FEATURE_YARN_3").is_some() {
+        // first try yarn2+, which uses "@yarnpkg/cli-dist" instead of "yarn"
+        let (url, mut index) = fetch_yarn_index("@yarnpkg/cli-dist")?;
 
+        if let Some(version) = index.tags.remove(&tag) {
+            debug!("Found yarn@{} matching tag '{}' from {}", version, tag, url);
+            if version.major == 2 {
+                return Err(ErrorKind::Yarn2NotSupported.into());
+            }
+            return Ok(version);
+        }
+        debug!(
+            "Did not find yarn matching tag '{}' from @yarnpkg/cli-dist",
+            tag
+        );
+    }
+
+    let (url, mut index) = fetch_yarn_index("yarn")?;
     match index.tags.remove(&tag) {
         Some(version) => {
             debug!("Found yarn@{} matching tag '{}' from {}", version, tag, url);
@@ -109,7 +116,40 @@ fn resolve_latest_legacy(url: String) -> Fallible<Version> {
 }
 
 fn resolve_semver_from_registry(matching: VersionReq) -> Fallible<Version> {
-    let (url, index) = fetch_yarn_index()?;
+    if env::var_os("VOLTA_FEATURE_YARN_3").is_some() {
+        // first try yarn2+, which uses "@yarnpkg/cli-dist" instead of "yarn"
+        let (url, index) = fetch_yarn_index("@yarnpkg/cli-dist")?;
+        let matching_entries: Vec<PackageDetails> = index
+            .entries
+            .into_iter()
+            .filter(|PackageDetails { version, .. }| matching.matches(version))
+            .collect();
+
+        if !matching_entries.is_empty() {
+            let details_opt = matching_entries
+                .iter()
+                .find(|PackageDetails { version, .. }| version.major >= 3);
+
+            match details_opt {
+                Some(details) => {
+                    debug!(
+                        "Found yarn@{} matching requirement '{}' from {}",
+                        details.version, matching, url
+                    );
+                    return Ok(details.version.clone());
+                }
+                None => {
+                    return Err(ErrorKind::Yarn2NotSupported.into());
+                }
+            }
+        }
+        debug!(
+            "Did not find yarn matching requirement '{}' from {}",
+            matching, url
+        );
+    }
+
+    let (url, index) = fetch_yarn_index("yarn")?;
 
     let details_opt = index
         .entries
@@ -124,6 +164,7 @@ fn resolve_semver_from_registry(matching: VersionReq) -> Fallible<Version> {
             );
             Ok(details.version)
         }
+        // at this point Yarn is not found in either registry
         None => Err(ErrorKind::YarnVersionNotFound {
             matching: matching.to_string(),
         }

tests/acceptance/corrupted_download.rs

@@ -1,4 +1,4 @@
-use crate::support::sandbox::{sandbox, DistroMetadata, NodeFixture, YarnFixture};
+use crate::support::sandbox::{sandbox, DistroMetadata, NodeFixture, Yarn1Fixture};
 use hamcrest2::assert_that;
 use hamcrest2::prelude::*;
 use semver::Version;
@@ -25,7 +25,7 @@ const NODE_VERSION_FIXTURES: [DistroMetadata; 2] = [
     },
 ];
 
-const YARN_VERSION_INFO: &str = r#"{
+const YARN_1_VERSION_INFO: &str = r#"{
     "name":"yarn",
     "dist-tags": { "latest": "1.2.42" },
     "versions": {
@@ -34,7 +34,7 @@ const YARN_VERSION_INFO: &str = r#"{
     }
 }"#;
 
-const YARN_VERSION_FIXTURES: [DistroMetadata; 2] = [
+const YARN_1_VERSION_FIXTURES: [DistroMetadata; 2] = [
     DistroMetadata {
         version: "0.0.1",
         compressed_size: 10,
@@ -81,9 +81,9 @@ fn install_valid_node_saves_to_inventory() {
 fn install_corrupted_yarn_leaves_inventory_unchanged() {
     let s = sandbox()
         .node_available_versions(NODE_VERSION_INFO)
-        .yarn_available_versions(YARN_VERSION_INFO)
+        .yarn_1_available_versions(YARN_1_VERSION_INFO)
         .distro_mocks::<NodeFixture>(&NODE_VERSION_FIXTURES)
-        .distro_mocks::<YarnFixture>(&YARN_VERSION_FIXTURES)
+        .distro_mocks::<Yarn1Fixture>(&YARN_1_VERSION_FIXTURES)
         .build();
 
     assert_that!(
@@ -99,9 +99,9 @@ fn install_valid_yarn_saves_to_inventory() {
     let s = sandbox()
         .platform(r#"{ "node": { "runtime": "1.2.3", "npm": null }, "yarn": null }"#)
         .node_available_versions(NODE_VERSION_INFO)
-        .yarn_available_versions(YARN_VERSION_INFO)
+        .yarn_1_available_versions(YARN_1_VERSION_INFO)
         .distro_mocks::<NodeFixture>(&NODE_VERSION_FIXTURES)
-        .distro_mocks::<YarnFixture>(&YARN_VERSION_FIXTURES)
+        .distro_mocks::<Yarn1Fixture>(&YARN_1_VERSION_FIXTURES)
         .build();
 
     assert_that!(