Implement pull

Author: Kobzol

Cargo.lock

@@ -3,9 +3,14 @@ name = "josh-sync"
 version = "0.1.0"
 edition = "2024"
 
+[[bin]]
+path = "src/bin/josh_sync.rs"
+name = "josh-sync"
+
 [dependencies]
 anyhow = "1"
 clap = { version = "4", features = ["derive"] }
+directories = "6"
 toml = "0.8"
 serde = { version = "1", features = ["derive"] }
 which = "8"

Cargo.toml

@@ -1,7 +1,8 @@
 use anyhow::Context;
 use clap::Parser;
-use josh_sync::JoshConfig;
+use josh_sync::config::{JoshConfig, load_config};
 use josh_sync::josh::{JoshProxy, try_install_josh};
+use josh_sync::sync::{GitSync, RustcPullError};
 use std::path::{Path, PathBuf};
 
 const DEFAULT_CONFIG_PATH: &str = "josh-sync.toml";
@@ -31,16 +32,31 @@ fn main() -> anyhow::Result<()> {
             let config = JoshConfig {
                 org: "rust-lang".to_string(),
                 repo: "<repository-name>".to_string(),
-                upstream_sha: None,
+                path: "<relative-subtree-path>".to_string(),
+                last_upstream_sha: None,
             };
-            let config = toml::to_string_pretty(&config).context("cannot serialize config")?;
-            std::fs::write(DEFAULT_CONFIG_PATH, config).context("cannot write config")?;
+            config
+                .write(Path::new(DEFAULT_CONFIG_PATH))
+                .context("cannot write config")?;
             println!("Created config file at {DEFAULT_CONFIG_PATH}");
         }
         Command::Pull { config } => {
             let config = load_config(&config)
                 .context("cannot load config. Run the `init` command to initialize it.")?;
             let josh = get_josh_proxy()?;
+            let sync = GitSync::new(config, josh);
+            if let Err(error) = sync.rustc_pull() {
+                match error {
+                    RustcPullError::NothingToPull => {
+                        eprintln!("Nothing to pull");
+                        std::process::exit(2);
+                    }
+                    RustcPullError::PullFailed(error) => {
+                        eprintln!("Pull failure: {error:?}");
+                        std::process::exit(1);
+                    }
+                }
+            }
         }
     }
 
@@ -65,10 +81,3 @@ fn get_josh_proxy() -> anyhow::Result<JoshProxy> {
         }
     }
 }
-
-fn load_config(path: &Path) -> anyhow::Result<JoshConfig> {
-    let data = std::fs::read_to_string(path)
-        .with_context(|| format!("cannot load config file from {}", path.display()))?;
-    let config: JoshConfig = toml::from_str(&data).context("cannot load config as TOML")?;
-    Ok(config)
-}

src/bin/josh_sync.rs

@@ -0,0 +1,46 @@
+use anyhow::Context;
+use std::path::{Path, PathBuf};
+
+#[derive(serde::Serialize, serde::Deserialize, Clone)]
+pub struct JoshConfig {
+    #[serde(default = "default_org")]
+    pub org: String,
+    pub repo: String,
+    /// Relative path where the subtree is located in rust-lang/rust.
+    /// For example `src/doc/rustc-dev-guide`.
+    pub path: String,
+    /// Last SHA of rust-lang/rust that was pulled into this subtree.
+    #[serde(default)]
+    pub last_upstream_sha: Option<String>,
+}
+
+impl JoshConfig {
+    pub fn full_repo_name(&self) -> String {
+        format!("{}/{}", self.org, self.repo)
+    }
+
+    pub fn write(&self, path: &Path) -> anyhow::Result<()> {
+        let config = toml::to_string_pretty(self).context("cannot serialize config")?;
+        std::fs::write(path, config).context("cannot write config")?;
+        Ok(())
+    }
+}
+
+pub struct JoshConfigWithPath {
+    pub config: JoshConfig,
+    pub path: PathBuf,
+}
+
+fn default_org() -> String {
+    String::from("rust-lang")
+}
+
+pub fn load_config(path: &Path) -> anyhow::Result<JoshConfigWithPath> {
+    let data = std::fs::read_to_string(path)
+        .with_context(|| format!("cannot load config file from {}", path.display()))?;
+    let config: JoshConfig = toml::from_str(&data).context("cannot load config as TOML")?;
+    Ok(JoshConfigWithPath {
+        config,
+        path: path.to_path_buf(),
+    })
+}

src/config.rs

@@ -1,5 +1,12 @@
+use crate::config::JoshConfig;
 use crate::utils::check_output;
+use anyhow::Context;
+use std::net::{SocketAddr, TcpStream};
 use std::path::PathBuf;
+use std::process::{Command, Stdio};
+use std::time::Duration;
+
+const JOSH_PORT: u16 = 42042;
 
 pub struct JoshProxy {
     path: PathBuf,
@@ -10,6 +17,47 @@ impl JoshProxy {
     pub fn lookup() -> Option<Self> {
         which::which("josh-proxy").ok().map(|path| Self { path })
     }
+
+    pub fn start(&self, config: &JoshConfig) -> anyhow::Result<RunningJoshProxy> {
+        // Determine cache directory.
+        let user_dirs =
+            directories::ProjectDirs::from("org", &config.full_repo_name(), "rustc-josh")
+                .context("cannot determine cache directory for Josh")?;
+        let local_dir = user_dirs.cache_dir().to_owned();
+
+        // Start josh, silencing its output.
+        let josh = std::process::Command::new(&self.path)
+            .arg("--local")
+            .arg(local_dir)
+            .args([
+                "--remote=https://github.com",
+                &format!("--port={JOSH_PORT}"),
+                "--no-background",
+            ])
+            .stdout(Stdio::null())
+            .stderr(Stdio::null())
+            .spawn()
+            .context("failed to start josh-proxy, make sure it is installed")?;
+
+        // Wait until the port is open. We try every 10ms until 1s passed.
+        for _ in 0..100 {
+            // This will generally fail immediately when the port is still closed.
+            let addr = SocketAddr::from(([127, 0, 0, 1], JOSH_PORT));
+            let josh_ready = TcpStream::connect_timeout(&addr, Duration::from_millis(1));
+
+            if josh_ready.is_ok() {
+                println!("josh up and running");
+                return Ok(RunningJoshProxy {
+                    process: josh,
+                    port: JOSH_PORT,
+                });
+            }
+
+            // Not ready yet.
+            std::thread::sleep(Duration::from_millis(10));
+        }
+        panic!("Even after waiting for 1s, josh-proxy is still not available.");
+    }
 }
 
 pub fn try_install_josh() -> Option<JoshProxy> {
@@ -22,6 +70,51 @@ pub fn try_install_josh() -> Option<JoshProxy> {
         "--tag",
         "r24.10.04",
         "josh-proxy",
-    ]);
+    ])
+    .expect("cannot install josh-proxy");
     JoshProxy::lookup()
 }
+
+/// Create a wrapper that represents a running instance of `josh-proxy` and stops it on drop.
+pub struct RunningJoshProxy {
+    process: std::process::Child,
+    port: u16,
+}
+
+impl RunningJoshProxy {
+    pub fn git_url(&self, upstream_repo: &str, commit: &str, filter: &str) -> String {
+        format!(
+            "http://localhost:{}/{upstream_repo}.git@{commit}{filter}.git",
+            self.port
+        )
+    }
+}
+
+impl Drop for RunningJoshProxy {
+    fn drop(&mut self) {
+        if cfg!(unix) {
+            // Try to gracefully shut it down.
+            Command::new("kill")
+                .args(["-s", "INT", &self.process.id().to_string()])
+                .output()
+                .expect("failed to SIGINT josh-proxy");
+            // Sadly there is no "wait with timeout"... so we just give it some time to finish.
+            std::thread::sleep(Duration::from_millis(100));
+            // Now hopefully it is gone.
+            if self
+                .process
+                .try_wait()
+                .expect("failed to wait for josh-proxy")
+                .is_some()
+            {
+                return;
+            }
+        }
+        // If that didn't work (or we're not on Unix), kill it hard.
+        eprintln!(
+            "I have to kill josh-proxy the hard way, let's hope this does not \
+            break anything."
+        );
+        self.process.kill().expect("failed to SIGKILL josh-proxy");
+    }
+}