Better support of authentication through SSH

Cargo.toml

@@ -71,7 +71,6 @@ mockall = "0.13"
 more-asserts = "0.3"
 once_cell = "1.20"
 oneshot = "0.1"
-openssh = "0.11"
 pin-project = "1"
 prometheus = "0.14"
 rand = "0.9"
@@ -93,6 +92,7 @@ serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 serde_repr = "0.1"
 sha2 = "0.10"
+shell-words = "1.1"
 shellexpand = "3.1"
 static_assertions = "1.1"
 sysinfo = "0.37"

git_xet/Cargo.toml

@@ -26,12 +26,10 @@ reqwest-middleware = { workspace = true }
 rust-netrc = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
+shell-words = { workspace = true }
 tempfile = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }
 
-[target.'cfg(unix)'.dependencies]
-openssh = { workspace = true }
-
 [dev-dependencies]
 serial_test = { workspace = true }

git_xet/src/auth.rs

@@ -127,7 +127,7 @@ pub fn get_credential(repo: &GitRepo, remote_url: &GitUrl, operation: Operation)
     // 5. check remote URL scheme
     if matches!(remote_url.scheme(), Scheme::Ssh | Scheme::GitSsh) {
         #[cfg(unix)]
-        return Ok(SSHCredentialHelper::new(remote_url, operation));
+        return Ok(SSHCredentialHelper::new(remote_url, repo, operation));
         #[cfg(not(unix))]
         return Err(GitXetError::not_supported(format!(
             "using {} in a repository with SSH Git URL is under development; please check back for 

git_xet/src/auth/ssh.rs

@@ -1,14 +1,16 @@
 use std::sync::Arc;
 
 use async_trait::async_trait;
-use hub_client::{CredentialHelper, HubClientError, Operation, Result};
-#[cfg(unix)]
-use openssh::{KnownHosts, Session};
+use hub_client::{CredentialHelper, Operation};
 use reqwest::header;
 use reqwest_middleware::RequestBuilder;
 use serde::Deserialize;
 
+use crate::errors::{GitXetError, Result};
+use crate::git_repo::GitRepo;
 use crate::git_url::GitUrl;
+use crate::utils::process_wrapping::run_program_captured_with_input_and_output;
+use crate::utils::ssh_connect::{SSHMetadata, get_sshcmd_and_args};
 
 #[derive(Deserialize)]
 struct GitLFSAuthentationResponseHeader {
@@ -32,39 +34,38 @@ struct GitLFSAuthenticateResponse {
 // it has a shorter TTL than that of a Xet CAS JWT.
 pub struct SSHCredentialHelper {
     remote_url: GitUrl,
+    repo: GitRepo,
     operation: Operation,
 }
 
 impl SSHCredentialHelper {
-    pub fn new(remote_url: &GitUrl, operation: Operation) -> Arc<Self> {
+    pub fn new(remote_url: &GitUrl, repo: &GitRepo, operation: Operation) -> Arc<Self> {
         Arc::new(Self {
             remote_url: remote_url.clone(),
+            repo: repo.clone(),
             operation,
         })
     }
 
-    #[cfg(unix)]
     async fn authenticate(&self) -> Result<GitLFSAuthenticateResponse> {
-        let host_url = self.remote_url.host_url().map_err(HubClientError::credential_helper_error)?;
-        let full_repo_path = self.remote_url.full_repo_path();
-        let session = Session::connect(&host_url, KnownHosts::Add)
-            .await
-            .map_err(HubClientError::credential_helper_error)?;
-
-        let output = session
-            .command("git-lfs-authenticate")
-            .arg(full_repo_path)
-            .arg(self.operation.as_str())
-            .output()
-            .await
-            .map_err(HubClientError::credential_helper_error)?;
-
-        serde_json::from_slice(&output.stdout).map_err(HubClientError::credential_helper_error)
-    }
+        let meta = SSHMetadata {
+            user_and_host: self.remote_url.user_and_host()?,
+            port: self.remote_url.port(),
+            arg_list: vec![
+                "git-lfs-authenticate".into(),
+                self.remote_url.full_repo_path(),
+                self.operation.as_str().into(),
+            ],
+        };
 
-    #[cfg(not(unix))]
-    async fn authenticate(&self) -> Result<GitLFSAuthenticateResponse> {
-        unimplemented!()
+        let (program, args) = get_sshcmd_and_args(&meta, &self.repo)?;
+
+        let (output, _err) =
+            run_program_captured_with_input_and_output(program, self.repo.git_path()?, args)?.wait_with_output()?;
+
+        let response: GitLFSAuthenticateResponse = serde_json::from_slice(&output).map_err(GitXetError::internal)?;
+
+        Ok(response)
     }
 }
 
@@ -86,14 +87,18 @@ mod tests {
     use hub_client::Operation;
 
     use super::SSHCredentialHelper;
+    use crate::git_repo::GitRepo;
     use crate::git_url::GitUrl;
+    use crate::test_utils::TestRepo;
 
     #[tokio::test]
     #[ignore = "need ssh server"]
     async fn test_ssh_cred_helper_local() -> Result<()> {
+        let test_repo = TestRepo::new("main")?;
+        let repo = GitRepo::open(test_repo.path())?;
         let remote_url = "ssh://git@localhost:2222/datasets/test/td";
         let parsed_url: GitUrl = remote_url.parse()?;
-        let ssh_helper = SSHCredentialHelper::new(&parsed_url, Operation::Download);
+        let ssh_helper = SSHCredentialHelper::new(&parsed_url, &repo, Operation::Download);
 
         let response = ssh_helper.authenticate().await?;
 
@@ -107,9 +112,11 @@ mod tests {
     #[tokio::test]
     #[ignore = "need ssh key"]
     async fn test_ssh_cred_helper_remote() -> Result<()> {
+        let test_repo = TestRepo::new("main")?;
+        let repo = GitRepo::open(test_repo.path())?;
         let remote_url = "ssh://[email protected]/seanses/tm"; // it seems that ssh port is not open on "huggingface.co"
         let parsed_url: GitUrl = remote_url.parse()?;
-        let ssh_helper = SSHCredentialHelper::new(&parsed_url, Operation::Upload);
+        let ssh_helper = SSHCredentialHelper::new(&parsed_url, &repo, Operation::Upload);
 
         let response = ssh_helper.authenticate().await?;
 

git_xet/src/git_url.rs

@@ -124,7 +124,8 @@ impl GitUrl {
     }
 
     // Returns the front part of the Git remote URL removing repo path,
-    // e.g. (scheme://)(auth)[host_name](:port)
+    // e.g. (scheme://)(auth@)[host_name](:port)
+    #[allow(unused)]
     pub fn host_url(&self) -> Result<String> {
         let scheme_str = if self.inner.scheme_prefix {
             format!("{}://", self.inner.scheme)
@@ -159,6 +160,29 @@ impl GitUrl {
         Ok(format!("{scheme_str}{auth_str}{host}{port_str}"))
     }
 
+    // Returns the user and host in the format of (auth@)[host].
+    pub fn user_and_host(&self) -> Result<String> {
+        let auth_str = match self.inner.scheme {
+            Scheme::Http | Scheme::Https | Scheme::Ssh | Scheme::GitSsh => {
+                match (&self.inner.user, &self.inner.token) {
+                    (Some(user), Some(token)) => format!("{}:{}@", user, token),
+                    (Some(user), None) => format!("{}@", user),
+                    (None, Some(token)) => format!("{}@", token),
+                    (None, None) => "".to_owned(),
+                }
+            },
+            _ => "".to_owned(),
+        };
+
+        let host = self
+            .inner
+            .host
+            .as_ref()
+            .ok_or_else(|| GitXetError::config_error("remote URL missing host name"))?;
+
+        Ok(format!("{auth_str}{host}"))
+    }
+
     pub fn port(&self) -> Option<u16> {
         self.inner.port
     }

git_xet/src/utils/mod.rs

@@ -1 +1,2 @@
 pub mod process_wrapping;
+pub mod ssh_connect;

git_xet/src/utils/process_wrapping.rs

@@ -33,7 +33,7 @@ where
 /// Return `Ok(())` if the command finishes correctly and the child's stdout and stderr are ignored;
 /// Return the underlying I/O error if the child process spawning or waiting fails; otherwise, the captured
 /// stdout and stderr of the child are wrapped in an `Err(GitXetError::CommandFailed(_))` and returned.
-#[allow(dead_code)]
+#[allow(unused)]
 pub fn run_program_captured<S1, P, I, S2>(program: S1, working_dir: P, args: I) -> Result<()>
 where
     S1: AsRef<OsStr>,
@@ -98,7 +98,6 @@ where
 ///
 /// let (response, _err) = cmd.wait_with_output()?;
 /// ```
-#[allow(dead_code)]
 pub fn run_program_captured_with_input_and_output<S1, P, I, S2>(
     program: S1,
     working_dir: P,