Merge #123

123: Refactor configure-toolchains r=MikailBag a=MikailBag



Co-authored-by: Mikail Bagishov <[email protected]>
Co-authored-by: Sergey Lisov <[email protected]>

Author: 3 people

Cargo.lock

@@ -68,7 +68,7 @@ fn copy_or_symlink_config(conf_params: &ConfigParams, params: &SetupParams) -> a
     } else {
         add(data_dir, "etc")?;
         add(data_dir, "etc/toolchains")?;
-        let cfg_dir_items = vec!["jjs.toml", "toolchains", "contest.toml"]
+        let cfg_dir_items = vec!["jjs.toml", "contest.toml"]
             .iter()
             .map(|x| cfg_dir.join(x))
             .collect();
@@ -158,7 +158,7 @@ fn setup_toolchains(params: &SetupParams) -> anyhow::Result<()> {
         None => return Ok(()),
     };
     cmd.arg(toolchain_spec_db_dir);
-    cmd.arg(target_dir.join("opt"));
+    cmd.arg(target_dir);
     cmd.arg("--trace")
         .arg(target_dir.join("configure-toolchains-log.txt"));
     let st = cmd.status()?.success();

src/setup/src/lib.rs

@@ -13,3 +13,5 @@ util = {path = "../util"}
 toml = "0.5.5"
 copy-ln = { git = "https://github.com/mikailbag/copy-ln", branch = "master" }
 tempfile = "3.1.0"
+tera = "0.11.20"
+error-chain = "0.12.1"  # used inderectly to render tera error into string 

src/soft/Cargo.toml

@@ -1,14 +1,16 @@
 use serde::Deserialize;
 
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Deserialize, Clone)]
 #[serde(rename_all = "kebab-case")]
 pub(crate) enum SetupKind {
     Trace,
 }
 
-#[derive(Debug, Deserialize)]
-pub(crate) struct Config {
+#[derive(Debug, Deserialize, Clone)]
+pub(crate) struct ToolchainConfig {
     kind: SetupKind,
+    #[serde(default)]
+    pub(crate) depends: Vec<String>,
+    #[serde(default)]
+    pub(crate) auto: bool,
 }
-
-impl Config {}

src/soft/src/config.rs

@@ -4,7 +4,8 @@ mod config;
 mod dep_collector;
 
 use crate::dep_collector::DepCollector;
-use anyhow::Context;
+use anyhow::{bail, Context};
+use error_chain::ChainedError;
 use std::{
     path::{Path, PathBuf},
     process::{Command, ExitCode},
@@ -13,8 +14,8 @@ use structopt::StructOpt;
 
 #[derive(StructOpt)]
 struct Options {
-    /// Spec files dir
-    spec: PathBuf,
+    /// Template files dir
+    tpls_dir: PathBuf,
     /// Out dir
     out: PathBuf,
     /// Trace log,
@@ -41,14 +42,18 @@ struct Options {
     print: Option<PathBuf>,
 }
 
-fn run_under_trace(script_path: &Path, data_path: &Path) -> anyhow::Result<Vec<u8>> {
+fn run_under_trace(
+    script_path: &Path,
+    data_path: &Path,
+    detect_out: &DetectScriptOutput,
+) -> anyhow::Result<Vec<u8>> {
     let current_dir = tempfile::TempDir::new().context("failed to create temp dir")?;
     println!("running in {}", current_dir.path().display());
     let log_out_file = current_dir.path().join("__jjs_trace.json");
     let data_path = data_path.canonicalize().context("data dir not exists")?;
     println!("script will use data from {}", data_path.display());
-    let status = Command::new("lxtrace")
-        .current_dir(current_dir.path())
+    let mut cmd = Command::new("lxtrace");
+    cmd.current_dir(current_dir.path())
         // machine-readable
         .arg("--json")
         // redirect to file, so it will not mix with script output
@@ -58,35 +63,214 @@ fn run_under_trace(script_path: &Path, data_path: &Path) -> anyhow::Result<Vec<u
         .arg("--")
         .arg("bash")
         .arg(script_path.canonicalize().context("script not exists")?)
-        .env("DATA", data_path)
-        .status()
-        .context("failed to start ktrace")?;
+        .env("DATA", data_path);
+    for (k, v) in &detect_out.env {
+        cmd.env(k, v);
+    }
+    let status = cmd.status().context("failed to start ktrace")?;
     if !status.success() {
         anyhow::bail!("ktrace returned error");
     }
     Ok(std::fs::read(&log_out_file).context("failed to read trace log")?)
 }
 
-fn process_toolchain(
-    dir: &Path,
+#[derive(Clone)]
+struct TemplateInfo {
+    dir: PathBuf,
+    name: String,
+    cfg: config::ToolchainConfig,
+}
+mod tpl_info_impls {
+    use super::*;
+    use std::{cmp::*, hash::*};
+
+    impl Hash for TemplateInfo {
+        fn hash<H: Hasher>(&self, hasher: &mut H) {
+            self.name.hash(hasher);
+        }
+    }
+
+    impl PartialEq for TemplateInfo {
+        fn eq(&self, that: &TemplateInfo) -> bool {
+            self.name == that.name
+        }
+    }
+
+    impl Eq for TemplateInfo {}
+}
+
+fn list_templates(dir: &Path) -> anyhow::Result<Vec<TemplateInfo>> {
+    let content = std::fs::read_dir(dir).context("failed to read toolchain templates dir")?;
+    let mut out = Vec::new();
+    for item in content {
+        let item = item.context("failed to stat toolchain template dir")?;
+        let cfg = item.path().join("config.toml");
+        let cfg = std::fs::read_to_string(cfg).context("failed to open manifest")?;
+        let cfg = toml::from_str(&cfg).context("failed to parse manifest")?;
+        let name = item
+            .path()
+            .file_name()
+            .unwrap()
+            .to_str()
+            .context("toolchain name is not utf8")?
+            .to_string();
+        out.push(TemplateInfo {
+            dir: item.path(),
+            name,
+            cfg,
+        });
+    }
+    Ok(out)
+}
+
+fn select_templates(
+    tpls: impl Iterator<Item = TemplateInfo>,
+    opt: &Options,
+) -> anyhow::Result<impl Iterator<Item = TemplateInfo>> {
+    let filter: Box<dyn FnMut(&TemplateInfo) -> bool> = if !opt.only.is_empty() {
+        Box::new(|tpl| opt.only.contains(&tpl.name) || tpl.cfg.auto)
+    } else if !opt.skip.is_empty() {
+        Box::new(|tpl| !opt.skip.contains(&tpl.name))
+    } else {
+        Box::new(|_tpl| true)
+    };
+    let tpls: Vec<_> = tpls.collect();
+    let roots: Vec<_> = tpls.clone().into_iter().filter(filter).collect();
+    let mut q = std::collections::HashSet::new();
+    let mut used = std::collections::HashSet::new();
+    q.extend(roots.into_iter());
+
+    while let Some(head) = q.iter().next() {
+        let tpl = head.clone();
+        q.remove(&tpl);
+        used.insert(tpl.clone());
+        for dep_name in &tpl.cfg.depends {
+            let dep = tpls
+                .iter()
+                .find(|d| d.name.as_str() == dep_name)
+                .context("dependency not found")?
+                .clone();
+            if !used.contains(&dep) {
+                q.insert(dep);
+            }
+        }
+    }
+    Ok(used.into_iter())
+}
+
+struct DetectScriptOutput {
+    env: std::collections::HashMap<String, String>,
+}
+
+impl std::str::FromStr for DetectScriptOutput {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> anyhow::Result<Self> {
+        let mut this = Self {
+            env: std::collections::HashMap::new(),
+        };
+        for line in s.lines() {
+            if line.starts_with("set-env:") {
+                let cmd = line.trim_start_matches("set-env:");
+                let parts: Vec<_> = cmd.splitn(2, '=').collect();
+                if parts.len() != 2 {
+                    bail!("set-env command does not look like var_name=var_value");
+                }
+                this.env.insert(parts[0].to_string(), parts[1].to_string());
+            } else {
+                bail!("unknown command: {}", line);
+            }
+        }
+        Ok(this)
+    }
+}
+
+fn run_detect_script(tpl: &TemplateInfo) -> anyhow::Result<Option<DetectScriptOutput>> {
+    let detect_script_path = tpl.dir.join("detect.sh");
+    if !detect_script_path.exists() {
+        bail!("detect.sh script missing");
+    }
+    let out_file_path = tempfile::NamedTempFile::new().context("failed to allocate temp file")?;
+
+    let status = std::process::Command::new("bash")
+        .arg(&detect_script_path)
+        .arg(out_file_path.path())
+        .status()
+        .context("failed to execute detect.sh script")?;
+    let script_out =
+        std::fs::read_to_string(out_file_path.path()).context("failed to read detect.sh output")?;
+    println!("--- script control output ---");
+    print!("{}", &script_out);
+    println!("--- end script control output ---");
+    let script_out = script_out
+        .parse()
+        .context("failed to parse detect.sh output")?;
+    let script_out = if status.success() {
+        Some(script_out)
+    } else {
+        None
+    };
+    Ok(script_out)
+}
+
+fn process_toolchain_invoke_conf(
+    tpl: &TemplateInfo,
+    out_dir: &Path,
+    detect_out: &DetectScriptOutput,
+) -> anyhow::Result<()> {
+    let out_file = out_dir
+        .join("etc/toolchains")
+        .join(format!("{}.toml", &tpl.name));
+    let in_file = tpl.dir.join("invoke-conf.toml");
+    if !in_file.exists() {
+        return Ok(());
+    }
+    let in_file = std::fs::read_to_string(&in_file).context("failed to read invoke-conf.toml")?;
+
+    let mut render_ctx = std::collections::HashMap::new();
+    for (k, v) in &detect_out.env {
+        let k = format!("env_{}", k);
+        render_ctx.insert(k, v.to_string());
+    }
+    let output = tera::Tera::one_off(&in_file, &render_ctx, false)
+        .map_err(|tera_err| {
+            // unfortunately, tera uses error_chain for error handling, which does not have `Sync` bound on its errors
+            // to workaround it, we render error to string
+            anyhow::anyhow!("{:#}", tera_err.display_chain())
+        })
+        .context("failed to render invoke config file")?;
+
+    std::fs::create_dir_all(out_file.parent().unwrap()).ok();
+
+    std::fs::write(&out_file, &output).context("failed to create config file")?;
+
+    Ok(())
+}
+
+fn process_toolchain_template(
+    tpl: TemplateInfo,
     collector: &mut DepCollector,
     mut event_log: Option<&mut dyn std::io::Write>,
+    out_dir: &Path,
 ) -> anyhow::Result<()> {
-    let manifest_path = dir.join("config.toml");
-    let manifest =
-        std::fs::read_to_string(manifest_path).context("config.toml not found or not readable")?;
-    let _manifest: config::Config = toml::from_str(&manifest).context("failed to parse config")?;
-    // TODO: look at config
-    let scripts: anyhow::Result<Vec<_>> = dir
-        .join("scripts")
+    let detect_out = match run_detect_script(&tpl)? {
+        Some(dso) => dso,
+        None => {
+            println!("Skipping toolchain {}: not available", &tpl.name);
+            return Ok(());
+        }
+    };
+    let scripts: anyhow::Result<Vec<_>> = tpl
+        .dir
+        .join("use")
         .read_dir()?
         .map(|item| item.map_err(|err| anyhow::Error::new(err).context("failed to read script")))
         .collect();
-    let current_dir = dir.join("data");
+    let current_dir = tpl.dir.join("data");
     for script in scripts? {
         println!("running {}", script.path().display());
-        let out =
-            run_under_trace(&script.path(), &current_dir).context("failed to collect trace")?;
+        let out = run_under_trace(&script.path(), &current_dir, &detect_out)
+            .context("failed to collect trace")?;
         let scanner = serde_json::Deserializer::from_slice(&out).into_iter();
         let mut cnt = 0;
         let mut cnt_items = 0;
@@ -124,6 +308,7 @@ fn process_toolchain(
             cnt_items, cnt, cnt_errors
         );
     }
+    process_toolchain_invoke_conf(&tpl, &out_dir, &detect_out)?;
     Ok(())
 }
 
@@ -138,32 +323,16 @@ fn main_inner() -> anyhow::Result<()> {
         }
         None => None,
     };
-    for item in std::fs::read_dir(&opt.spec).context("failed read spec dir")? {
-        let item = item.context("failed read spec dir")?;
-        let title = item
-            .path()
-            .file_name()
-            .unwrap()
-            .to_str()
-            .context("toolchain name is not utf8")?
-            .to_string();
+    let templates = list_templates(&opt.tpls_dir)?;
+    let templates = select_templates(templates.into_iter(), &opt)?;
+    for tpl in templates {
+        println!("------ processing {} ------", &tpl.name);
 
-        let mut ok = true;
-        if !opt.only.is_empty() {
-            ok = opt.only.contains(&title);
-        } else if opt.skip.contains(&title) {
-            ok = false;
-        }
-        if ok {
-            println!("processing {}", &title);
-        } else {
-            println!("skipping {}", &title);
-            continue;
-        }
-        if let Err(e) = process_toolchain(
-            &item.path(),
+        if let Err(e) = process_toolchain_template(
+            tpl,
             &mut collector,
             log_file.as_mut().map(|x| x as _),
+            &opt.out,
         ) {
             util::print_error(&*e);
         }
@@ -172,6 +341,7 @@ fn main_inner() -> anyhow::Result<()> {
         "all toolchains processed: {} files found",
         collector.count()
     );
+    let toolchain_files_output_dir = opt.out.join("opt");
     let mut process_file: Box<dyn FnMut(&str)> = if let Some(path) = &opt.print {
         let out_file = std::fs::File::create(&path).context("failed to open output file")?;
         let mut out_file = std::io::BufWriter::new(out_file);
@@ -185,7 +355,9 @@ fn main_inner() -> anyhow::Result<()> {
             if file.is_dir() && !opt.copy_dirs {
                 return;
             }
-            if let Err(e) = copy_ln::copy(&opt.out, file, true, !opt.copy_symlinks) {
+            if let Err(e) =
+                copy_ln::copy(&toolchain_files_output_dir, file, true, !opt.copy_symlinks)
+            {
                 eprintln!("{:?}", e);
             }
         })