add parallelism to autorust using rayon (#1416)

Author: demoray

services/autorust/azure-autorust/Cargo.toml

@@ -8,3 +8,4 @@ publish = false
 [dependencies]
 autorust_codegen = { path = "../codegen" }
 clap = { version = "4.0", features = ["derive"] }
+rayon = "1.8"

services/autorust/azure-autorust/src/main.rs

@@ -6,9 +6,10 @@ use autorust_codegen::{
     crates::{list_crate_names, list_dirs},
     gen, get_mgmt_readmes, get_svc_readmes,
     jinja::{CheckAllServicesYml, PublishSdksYml, PublishServicesYml, WorkspaceCargoToml},
-    Error, ErrorKind, Result, RunConfig,
+    ErrorKind, Result, ResultExt, RunConfig, SpecReadme,
 };
 use clap::Parser;
+use rayon::prelude::*;
 
 #[derive(Debug, clap::Parser)]
 struct Args {
@@ -43,29 +44,47 @@ fn main() -> Result<()> {
             gen_workflow_publish_services()?;
         }
     }
-    if args.fmt {
-        fmt(packages)?;
-    }
     Ok(())
 }
 
-fn gen_crates(only_packages: &[&str]) -> Result<()> {
-    let svc = get_svc_readmes()?.into_iter().map(|x| ("svc", x));
-    let mgmt = get_mgmt_readmes()?.into_iter().map(|x| ("mgmt", x));
-
-    for (i, (crate_type, spec)) in svc.chain(mgmt).enumerate() {
-        let output_folder = format!("../{crate_type}");
-        let prefix = format!("azure_{crate_type}_");
+fn gen_crate(only_packages: &[&str], crate_type: &str, spec: &SpecReadme) -> Result<Option<(String, Vec<String>)>> {
+    let output_folder = format!("../{crate_type}");
+    let prefix = format!("azure_{crate_type}_");
 
-        let run_config = RunConfig::new(&prefix);
-        let package_name = gen::package_name(&spec, &run_config);
-        if !only_packages.is_empty() && !only_packages.contains(&package_name.as_str()) {
-            continue;
-        }
+    let run_config = RunConfig::new(&prefix);
+    let package_name = gen::package_name(spec, &run_config);
 
-        println!("{package_name} ({i})");
-        gen::gen_crate(&package_name, &spec, &run_config, &output_folder)?;
+    if !only_packages.is_empty() && !only_packages.contains(&package_name.as_str()) {
+        Ok(None)
+    } else {
+        let tags = gen::gen_crate(&package_name, spec, &run_config, &output_folder)
+            .with_context(ErrorKind::CodeGen, || format!("generating {package_name}"))?;
+        Ok(Some((package_name, tags)))
     }
+}
+
+fn gen_crates(only_packages: &[&str]) -> Result<()> {
+    let svc = get_svc_readmes()?.into_iter().map(|x| ("svc".to_string(), x));
+    let mgmt = get_mgmt_readmes()?.into_iter().map(|x| ("mgmt".to_string(), x));
+    let crate_iters = svc.chain(mgmt).collect::<Vec<_>>();
+
+    let results: Result<Vec<_>> = crate_iters
+        .par_iter()
+        .map(|(crate_type, spec)| gen_crate(only_packages, crate_type, spec))
+        .collect::<Vec<_>>()
+        .into_iter()
+        .collect();
+
+    (results?).into_iter().flatten().for_each(|(package_name, tags)| {
+        println!("{package_name}");
+        if tags.is_empty() {
+            println!("  No tags");
+        } else {
+            for tag in tags {
+                println!("- {tag}");
+            }
+        }
+    });
 
     Ok(())
 }
@@ -124,22 +143,3 @@ fn gen_workflow_publish_services() -> Result<()> {
     yml.create("../../.github/workflows/publish-services.yml")?;
     Ok(())
 }
-
-/// Run `cargo fmt` on the services workspace or a subset of packages.
-fn fmt(only_packages: &[&str]) -> Result<()> {
-    let services_dir = "../";
-    let mut args = vec!["fmt"];
-    if !only_packages.is_empty() {
-        args.push("-p");
-        for package in only_packages {
-            args.push(package);
-        }
-    }
-    let out = std::process::Command::new("cargo").current_dir(services_dir).args(args).output()?;
-    if !out.status.success() {
-        println!("cargo fmt failed");
-        println!("{}", std::str::from_utf8(&out.stderr)?);
-        return Err(Error::new(ErrorKind::Io, "cargo fmt failed"));
-    }
-    Ok(())
-}

services/autorust/codegen/src/codegen_operations.rs

@@ -1213,7 +1213,7 @@ impl ToTokens for RequestBuilderIntoFutureCode {
                             },
                         )
                     } else {
-                        println!("unsupported LRO: {lro_options:?}");
+                        // println!("unsupported LRO: {lro_options:?}");
 
                         (
                             quote! {

services/autorust/codegen/src/gen.rs

@@ -1,7 +1,7 @@
 use crate::{
     autorust_toml, cargo_toml, io, lib_rs,
     readme_md::{self, ReadmeMd},
-    CrateConfig, Error, Result, RunConfig, SpecReadme,
+    run, CrateConfig, Error, Result, RunConfig, SpecReadme,
 };
 use std::{collections::HashMap, fs};
 
@@ -11,7 +11,8 @@ pub fn package_name(spec: &SpecReadme, run_config: &RunConfig) -> String {
     format!("{}{}", &run_config.crate_name_prefix, &spec.service_name())
 }
 
-pub fn gen_crate(package_name: &str, spec: &SpecReadme, run_config: &RunConfig, output_folder: &str) -> Result<()> {
+pub fn gen_crate(package_name: &str, spec: &SpecReadme, run_config: &RunConfig, output_folder: &str) -> Result<Vec<String>> {
+    let mut generated_tags = vec![];
     let spec_config = spec.config()?;
     let service_name = &spec.service_name();
     let output_folder = &io::join(output_folder, service_name)?;
@@ -43,16 +44,15 @@ pub fn gen_crate(package_name: &str, spec: &SpecReadme, run_config: &RunConfig,
 
     let tags = &package_config.filter_tags(spec_config.tags());
     if tags.is_empty() {
-        println!("not generating {} - no tags", spec.spec());
-        return Ok(());
+        return Ok(generated_tags);
     }
 
     let mut operation_totals = HashMap::new();
     let mut api_version_totals = HashMap::new();
     let mut api_versions = HashMap::new();
     let mut has_xml = false;
     for tag in tags {
-        println!("  {}", tag.name());
+        generated_tags.push(tag.name().to_owned());
         let output_folder = io::join(&src_folder, tag.rust_mod_name())?;
         let input_files: Result<Vec<_>> = tag
             .input_files()
@@ -65,7 +65,7 @@ pub fn gen_crate(package_name: &str, spec: &SpecReadme, run_config: &RunConfig,
             output_folder,
             input_files,
         };
-        let cg = crate::run(crate_config, &package_config)?;
+        let cg = run(crate_config, &package_config)?;
         let operations = cg.spec.operations()?;
         operation_totals.insert(tag.name(), operations.len());
         let mut versions = cg.spec.api_versions();
@@ -95,5 +95,5 @@ pub fn gen_crate(package_name: &str, spec: &SpecReadme, run_config: &RunConfig,
     };
     readme.create(&readme_path)?;
 
-    Ok(())
+    Ok(generated_tags)
 }

services/autorust/codegen/src/lib.rs

@@ -15,16 +15,18 @@ pub mod lib_rs;
 pub mod readme_md;
 pub mod spec;
 mod status_codes;
+
 use autorust_toml::PackageConfig;
 use camino::{Utf8Path, Utf8PathBuf};
 use config_parser::Configuration;
 pub use error::{Error, ErrorKind, Result, ResultExt};
 use proc_macro2::TokenStream;
 use quote::ToTokens;
-use std::io::Write;
 use std::{
     collections::HashSet,
     fs::{self, File},
+    io::Write,
+    process::{Command, Stdio},
 };
 
 pub use self::{
@@ -124,15 +126,40 @@ pub fn run<'a>(crate_config: &'a CrateConfig, package_config: &'a PackageConfig)
     Ok(cg)
 }
 
+fn rustfmt(input: &str) -> Result<Vec<u8>> {
+    let mut cmd = Command::new("rustfmt");
+    cmd.args(["--emit", "stdout", "--edition", "2021"]);
+    let mut child = cmd
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .spawn()
+        .with_context(ErrorKind::Io, || "spawn rustfmt")?;
+    child
+        .stdin
+        .as_mut()
+        .ok_or_else(|| Error::message(ErrorKind::Io, "rustfmt stdin is not available"))?
+        .write_all(input.as_bytes())
+        .with_context(ErrorKind::Io, || "write to rustfmt stdin")?;
+    let output = child.wait_with_output().with_context(ErrorKind::Io, || "wait for rustfmt")?;
+    if !output.status.success() {
+        return Err(Error::message(
+            ErrorKind::Io,
+            format!("rustfmt failed: {}", String::from_utf8_lossy(&output.stderr)),
+        ));
+    }
+    Ok(output.stdout)
+}
+
 fn write_file<P: AsRef<Utf8Path>>(file: P, tokens: &TokenStream, print_writing_file: bool) -> Result<()> {
     let file = file.as_ref();
     if print_writing_file {
         println!("writing file {}", &file);
     }
     let code = tokens.to_string();
+    let code = rustfmt(&code)?;
     let mut buffer = File::create(file).with_context(ErrorKind::Io, || format!("create file {file}"))?;
     buffer
-        .write_all(code.as_bytes())
+        .write_all(&code)
         .with_context(ErrorKind::Io, || format!("write file {file}"))?;
     Ok(())
 }