feat: patchable init mirroring

Cargo.toml

@@ -13,3 +13,4 @@ toml = "0.8.19"
 tracing = "0.1.41"
 tracing-indicatif = "0.3.9"
 tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }
+url = "2.5.4"

rust/patchable/Cargo.toml

@@ -14,3 +14,4 @@ toml.workspace = true
 tracing.workspace = true
 tracing-indicatif.workspace = true
 tracing-subscriber.workspace = true
+url.workspace = true

rust/patchable/README.md

@@ -31,18 +31,30 @@ For more details, run `cargo patchable --help`.
 
 ## Configuration
 
-Patchable stores a per-version file in `docker-images/<PRODUCT>/stackable/patches/<VERSION>/patchable.toml`.
-It currently recognizes the following keys:
+Patchable uses a two-level configuration system:
 
+1. A product-level config file at `docker-images/<PRODUCT>/stackable/patches/patchable.toml`
+2. A version-level config file at `docker-images/<PRODUCT>/stackable/patches/<VERSION>/patchable.toml`
+
+The product-level config contains:
 - `upstream` - the URL of the upstream repository (such as `https://github.com/apache/druid.git`)
-- `base` - the commit hash of the upstream base commit (such as `7cffb81a8e124d5f218f9af16ad685acf5e9c67c`)
+- `mirror` - optional URL of a mirror repository (such as `https://github.com/stackabletech/druid.git`)
 
-### Template
+The version-level config contains:
+- `base` - the commit hash of the upstream base commit
 
-Instead of creating this manually, run `patchable init`:
+### Template
 
+If you're adding a completely new product, you need to create the product-level config once:
 ```toml
-cargo patchable init druid 28.0.0 --upstream=https://github.com/apache/druid.git --base=druid-28.0.0
+# docker-images/druid/stackable/patches/patchable.toml
+upstream = "https://github.com/apache/druid.git"
+mirror = "https://github.com/stackabletech/druid.git"
+```
+
+If you just want to add a new version, initiatilize the version-level config with patchable:
+```
+cargo patchable init druid 28.0.0 --base=druid-28.0.0
 ```
 
 ## Glossary

rust/patchable/src/main.rs

@@ -9,10 +9,12 @@ use std::{fs::File, io::Write, path::PathBuf};
 
 use git2::{Oid, Repository};
 use serde::{Deserialize, Serialize};
-use snafu::{ensure, OptionExt, ResultExt as _, Snafu};
+use snafu::{OptionExt, ResultExt as _, Snafu};
 use tracing_indicatif::IndicatifLayer;
 use tracing_subscriber::{layer::SubscriberExt as _, util::SubscriberInitExt as _};
 
+use crate::utils::setup_git_credentials;
+
 #[derive(clap::Parser)]
 struct ProductVersion {
     /// The product name slug (such as druid)
@@ -24,12 +26,37 @@ struct ProductVersion {
     version: String,
 }
 
+/// Configuration that applies to all versions of a product, located at $ROOT/$product/stackable/patches/patchable.toml
+///
+/// Must be created by hand (for now).
 #[derive(Deserialize, Serialize)]
-struct ProductVersionConfig {
+#[serde(rename_all = "kebab-case")]
+struct ProductConfig {
+    /// The upstream repository URL
     upstream: String,
+
+    /// The repository that commits are mirrored to by `init --mirror`, typically `https://github.com/stackabletech/$product.git`
+    ///
+    /// This value is _not_ used by `checkout`, that uses [`ProductVersionConfig::mirror`] instead.
+    /// `init --mirror` copies this value into [`ProductVersionConfig::mirror`].
+    default_mirror: Option<String>,
+}
+
+/// Configuration that applies to an individual version of a product, located at $ROOT/$product/stackable/patches/$version/patchable.toml
+///
+/// Typically created by `patchable init`.
+#[derive(Deserialize, Serialize)]
+struct ProductVersionConfig {
+    /// The mirror repository that this version can be fetched from
+    ///
+    /// Copied from [`ProductConfig::default_mirror`] by `init --mirror`.
+    mirror: Option<String>,
+
+    /// The upstream base commit for this version
+    ///
+    /// Must be a commit ID, not a generic commitish (branch, tag, or anotherother ref), those are resolved by `init`.
     #[serde(with = "utils::oid_serde")]
     base: Oid,
-    mirror: Option<String>,
 }
 
 struct ProductVersionContext {
@@ -38,16 +65,24 @@ struct ProductVersionContext {
 }
 
 impl ProductVersionContext {
-    fn load_config(&self) -> Result<ProductVersionConfig> {
-        let path = &self.config_path();
+    fn load_config<T: for<'de> Deserialize<'de>>(&self, path: &PathBuf) -> Result<T> {
         tracing::info!(
             config.path = ?path,
             "loading config"
         );
-        toml::from_str::<ProductVersionConfig>(
-            &std::fs::read_to_string(path).context(LoadConfigSnafu { path })?,
-        )
-        .context(ParseConfigSnafu { path })
+
+        toml::from_str::<T>(&std::fs::read_to_string(path).context(LoadConfigSnafu { path })?)
+            .context(ParseConfigSnafu { path })
+    }
+
+    fn load_product_config(&self) -> Result<ProductConfig> {
+        let path = self.product_config_path();
+        self.load_config(&path)
+    }
+
+    fn load_version_config(&self) -> Result<ProductVersionConfig> {
+        let path = self.version_config_path();
+        self.load_config(&path)
     }
 
     /// The root directory for files related to the product (across all versions).
@@ -63,10 +98,15 @@ impl ProductVersionContext {
     }
 
     /// The patchable configuration file for the product version.
-    fn config_path(&self) -> PathBuf {
+    fn version_config_path(&self) -> PathBuf {
         self.patch_dir().join("patchable.toml")
     }
 
+    /// The product-level patchable configuration file
+    fn product_config_path(&self) -> PathBuf {
+        self.product_dir().join("stackable/patches/patchable.toml")
+    }
+
     /// The directory containing all ephemeral data used by patchable for the product (across all versions).
     ///
     /// Should be gitignored, and can safely be deleted as long as all relevant versions have been `patchable export`ed.
@@ -126,6 +166,10 @@ enum Cmd {
         /// Check out the base commit, without applying patches
         #[clap(long)]
         base_only: bool,
+
+        /// Use SSH for git operations
+        #[clap(long)]
+        ssh: bool,
     },
 
     /// Export the patches from the source tree at docker-images/<PRODUCT>/patchable-work/worktree/<VERSION>
@@ -141,20 +185,19 @@ enum Cmd {
         #[clap(flatten)]
         pv: ProductVersion,
 
-        /// The upstream URL (such as https://github.com/apache/druid.git)
-        #[clap(long)]
-        upstream: String,
-
         /// The upstream commit-ish (such as druid-28.0.0) that the patch series applies to
         ///
         /// Refs (such as tags and branches) will be resolved to commit IDs.
         #[clap(long)]
         base: String,
 
-        /// Assume a mirror exists at stackabletech/<repo_name> and push the base ref to it.
-        /// The mirror URL will be stored in patchable.toml instead of the original upstream.
+        /// Mirror the product version to the default mirror repository
+        #[clap(long)]
+        mirror: bool,
+
+        /// Use SSH for git operations
         #[clap(long)]
-        mirrored: bool,
+        ssh: bool,
     },
 
     /// Shows the patch directory for a given product version
@@ -203,8 +246,13 @@ pub enum Error {
         path: PathBuf,
     },
 
-    #[snafu(display("failed to parse upstream URL {url:?} to extract repository name"))]
-    ParseUpstreamUrl { url: String },
+    #[snafu(display("failed to rewrite URL for SSH: {source}"))]
+    UrlRewrite { source: utils::UrlRewriteError },
+
+    #[snafu(display(
+        "mirroring requested, but default-mirror is not configured in product configuration"
+    ))]
+    InitMirrorNotConfigured,
     #[snafu(display("failed to add temporary mirror remote for {url:?}"))]
     AddMirrorRemote { source: git2::Error, url: String },
     #[snafu(display("failed to push commit {commit} (as {refspec}) to mirror {url:?}"))]
@@ -292,20 +340,31 @@ fn main() -> Result<()> {
         }
     };
     match opts.cmd {
-        Cmd::Checkout { pv, base_only } => {
+        Cmd::Checkout { pv, base_only, ssh } => {
             let ctx = ProductVersionContext {
                 pv,
                 images_repo_root,
             };
-            let config = ctx.load_config()?;
+            let product_config = ctx.load_product_config()?;
+            let version_config = ctx.load_version_config()?;
             let product_repo_root = ctx.product_repo();
             let product_repo = repo::ensure_bare_repo(&product_repo_root)
                 .context(OpenProductRepoForCheckoutSnafu)?;
 
+            let mut upstream = version_config.mirror.unwrap_or_else(|| {
+                    tracing::warn!("this product version is not mirrored, re-init it with --mirror before merging it");
+                    product_config.upstream
+                });
+
+            if ssh {
+                upstream =
+                    utils::rewrite_git_https_url_to_ssh(&upstream).context(UrlRewriteSnafu)?;
+            }
+
             let base_commit = repo::resolve_and_fetch_commitish(
                 &product_repo,
-                &config.base.to_string(),
-                config.mirror.as_deref().unwrap_or(&config.upstream),
+                &version_config.base.to_string(),
+                &upstream,
             )
             .context(FetchBaseCommitSnafu)?;
             let base_branch = ctx.base_branch();
@@ -366,7 +425,7 @@ fn main() -> Result<()> {
                 pv,
                 images_repo_root,
             };
-            let config = ctx.load_config()?;
+            let config = ctx.load_version_config()?;
 
             let product_worktree_root = ctx.worktree_root();
             tracing::info!(
@@ -417,9 +476,9 @@ fn main() -> Result<()> {
 
         Cmd::Init {
             pv,
-            upstream,
             base,
-            mirrored,
+            mirror,
+            ssh,
         } => {
             let ctx = ProductVersionContext {
                 pv,
@@ -434,77 +493,81 @@ fn main() -> Result<()> {
             .in_scope(|| repo::ensure_bare_repo(&product_repo_root))
             .context(OpenProductRepoForCheckoutSnafu)?;
 
+            let config = ctx.load_product_config()?;
+            let upstream = if ssh {
+                utils::rewrite_git_https_url_to_ssh(&config.upstream).context(UrlRewriteSnafu)?
+            } else {
+                config.upstream
+            };
+
             // --base can be a reference, but patchable.toml should always have a resolved commit id,
             // so that it cannot be changed under our feet (without us knowing so, anyway...).
             tracing::info!(?base, "resolving base commit-ish");
-            let base_commit = repo::resolve_and_fetch_commitish(&product_repo, &base, &upstream).context(FetchBaseCommitSnafu)?;
-            let mut upstream_mirror = None;
-
-            if mirrored {
-                // Parse e.g. "https://github.com/apache/druid.git" into "druid"
-                let repo_name = upstream.split('/').last().map(|repo| repo.trim_end_matches(".git")).context(ParseUpstreamUrlSnafu { url: &upstream })?;
-
-                ensure!(!repo_name.is_empty(), ParseUpstreamUrlSnafu { url: &upstream });
-
-                let mirror_url = format!("https://github.com/stackabletech/{}.git", repo_name);
-                tracing::info!(%mirror_url, "using mirror repository");
+            let base_commit = repo::resolve_and_fetch_commitish(&product_repo, &base, &upstream)
+                .context(FetchBaseCommitSnafu)?;
+            tracing::info!(?base, base.commit = ?base_commit, "resolved base commit");
 
+            let mirror_url = if mirror {
+                let mut mirror_url = config
+                    .default_mirror
+                    .context(InitMirrorNotConfiguredSnafu)?;
+                if ssh {
+                    mirror_url =
+                        utils::rewrite_git_https_url_to_ssh(&mirror_url).context(UrlRewriteSnafu)?
+                };
                 // Add mirror remote
-                let mut mirror_remote = product_repo
-                    .remote_anonymous(&mirror_url)
-                    .context(AddMirrorRemoteSnafu { url: mirror_url.clone() })?;
+                let mut mirror_remote =
+                    product_repo
+                        .remote_anonymous(&mirror_url)
+                        .context(AddMirrorRemoteSnafu {
+                            url: mirror_url.clone(),
+                        })?;
 
                 // Push the base commit to the mirror
                 tracing::info!(commit = %base_commit, base = base, url = mirror_url, "pushing commit to mirror");
-                let mut callbacks = git2::RemoteCallbacks::new();
-                callbacks.credentials(|_url, username_from_url, _allowed_types| {
-                    git2::Cred::credential_helper(
-                        &git2::Config::open_default().unwrap(), // Use default git config
-                        _url,
-                        username_from_url,
-                    )
-                });
+                let mut callbacks = setup_git_credentials();
 
                 // Add progress tracking for push operation
-                let (span_push, mut quant_push) = utils::setup_progress_tracking(tracing::info_span!("pushing"));
+                let (span_push, mut quant_push) =
+                    utils::setup_progress_tracking(tracing::info_span!("pushing"));
+                let _ = span_push.enter();
+
                 callbacks.push_transfer_progress(move |current, total, _| {
                     if total > 0 {
-                        quant_push.update_span_progress(
-                            current,
-                            total,
-                            &span_push,
-                        );
+                        quant_push.update_span_progress(current, total, &span_push);
                     }
                 });
 
                 let mut push_options = git2::PushOptions::new();
                 push_options.remote_callbacks(callbacks);
 
-                let refspec = format!("{}:refs/tags/{}", base_commit, base);
-                tracing::info!(refspec = refspec, "constructed push refspec");
+                // Always push the commit as a Git tag named like the value of `base`
+                let refspec = format!("{base_commit}:refs/tags/{base}");
+                tracing::info!(refspec, "constructed push refspec");
 
                 mirror_remote
                     .push(&[&refspec], Some(&mut push_options))
                     .context(PushToMirrorSnafu {
-                        url: mirror_url.clone(),
+                        url: &mirror_url,
                         refspec: &refspec,
                         commit: base_commit,
                     })?;
 
                 tracing::info!("successfully pushed base ref to mirror");
-
-                upstream_mirror = Some(mirror_url);
+                Some(mirror_url)
+            } else {
+                tracing::warn!(
+                    "this version is not mirrored, re-run with --mirror before merging into main"
+                );
+                None
             };
 
-            tracing::info!(?base, base.commit = ?base_commit, "resolved base commit");
-
-            tracing::info!("saving configuration");
+            tracing::info!("saving version-level configuration");
             let config = ProductVersionConfig {
-                upstream,
-                mirror: upstream_mirror,
                 base: base_commit,
+                mirror: mirror_url,
             };
-            let config_path = ctx.config_path();
+            let config_path = ctx.version_config_path();
             if let Some(config_dir) = config_path.parent() {
                 std::fs::create_dir_all(config_dir)
                     .context(CreatePatchDirSnafu { path: config_dir })?;