Merge #3738

3738: Implement ra_proc_macro client logic r=matklad a=edwin0cheng

This PR add the actual client logic for `ra_proc_macro` crate:

1. Define all necessary rpc serialization data structure, which include `ra_tt` related data and some task messages. Although adding `Serialize` and `Deserialize` trait to ra_tt directly seem to be much easier,   we deliberately duplicate the `ra_tt` struct with `#[serde(with = "XXDef")]` for separation of  code responsibility. 
2. Define a simplified version of lsp base protocol for rpc, which basically copy from lsp-server code base. 
3. Implement the actual `IO` for the client side progress spawning and message passing.

Co-authored-by: Edwin Cheng <[email protected]>

Author: bors[bot]

Cargo.lock

@@ -9,6 +9,15 @@ pub struct ProcMacroExpander {
     proc_macro_id: ProcMacroId,
 }
 
+macro_rules! err {
+    ($fmt:literal, $($tt:tt),*) => {
+        mbe::ExpandError::ProcMacroError(tt::ExpansionError::Unknown(format!($fmt, $($tt),*)))
+    };
+    ($fmt:literal) => {
+        mbe::ExpandError::ProcMacroError(tt::ExpansionError::Unknown($fmt.to_string()))
+    }
+}
+
 impl ProcMacroExpander {
     pub fn new(krate: CrateId, proc_macro_id: ProcMacroId) -> ProcMacroExpander {
         ProcMacroExpander { krate, proc_macro_id }
@@ -25,8 +34,24 @@ impl ProcMacroExpander {
             .proc_macro
             .get(self.proc_macro_id.0 as usize)
             .clone()
-            .ok_or_else(|| mbe::ExpandError::ConversionError)?;
+            .ok_or_else(|| err!("No derive macro found."))?;
+
+        let tt = remove_derive_atr(tt, &proc_macro.name)
+            .ok_or_else(|| err!("Fail to remove derive for custom derive"))?;
 
         proc_macro.expander.expand(&tt, None).map_err(mbe::ExpandError::from)
     }
 }
+
+fn remove_derive_atr(tt: &tt::Subtree, _name: &str) -> Option<tt::Subtree> {
+    // FIXME: proper handle the remove derive
+    // We assume the first 2 tokens are #[derive(name)]
+    if tt.token_trees.len() > 2 {
+        let mut tt = tt.clone();
+        tt.token_trees.remove(0);
+        tt.token_trees.remove(0);
+        return Some(tt);
+    }
+
+    None
+}

crates/ra_hir_expand/src/proc_macro.rs

@@ -10,3 +10,8 @@ doctest = false
 
 [dependencies]
 ra_tt = { path = "../ra_tt" }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+log = "0.4.8"
+crossbeam-channel = "0.4.0"
+jod-thread = "0.1.1"

crates/ra_proc_macro/Cargo.toml

@@ -5,55 +5,104 @@
 //! is used to provide basic infrastructure  for communication between two
 //! processes: Client (RA itself), Server (the external program)
 
+mod rpc;
+mod process;
+pub mod msg;
+
+use process::{ProcMacroProcessSrv, ProcMacroProcessThread};
 use ra_tt::{SmolStr, Subtree};
+use rpc::ProcMacroKind;
 use std::{
     path::{Path, PathBuf},
     sync::Arc,
 };
 
-#[derive(Debug, Clone, PartialEq, Eq)]
+pub use rpc::{ExpansionResult, ExpansionTask};
+
+#[derive(Debug, Clone)]
 pub struct ProcMacroProcessExpander {
     process: Arc<ProcMacroProcessSrv>,
+    dylib_path: PathBuf,
     name: SmolStr,
 }
 
+impl Eq for ProcMacroProcessExpander {}
+impl PartialEq for ProcMacroProcessExpander {
+    fn eq(&self, other: &Self) -> bool {
+        self.name == other.name
+            && self.dylib_path == other.dylib_path
+            && Arc::ptr_eq(&self.process, &other.process)
+    }
+}
+
 impl ra_tt::TokenExpander for ProcMacroProcessExpander {
     fn expand(
         &self,
-        _subtree: &Subtree,
+        subtree: &Subtree,
         _attr: Option<&Subtree>,
     ) -> Result<Subtree, ra_tt::ExpansionError> {
-        // FIXME: do nothing for now
-        Ok(Subtree::default())
+        self.process.custom_derive(&self.dylib_path, subtree, &self.name)
     }
 }
 
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ProcMacroProcessSrv {
-    path: PathBuf,
+#[derive(Debug)]
+enum ProcMacroClientKind {
+    Process { process: Arc<ProcMacroProcessSrv>, thread: ProcMacroProcessThread },
+    Dummy,
 }
 
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum ProcMacroClient {
-    Process { process: Arc<ProcMacroProcessSrv> },
-    Dummy,
+#[derive(Debug)]
+pub struct ProcMacroClient {
+    kind: ProcMacroClientKind,
 }
 
 impl ProcMacroClient {
-    pub fn extern_process(process_path: &Path) -> ProcMacroClient {
-        let process = ProcMacroProcessSrv { path: process_path.into() };
-        ProcMacroClient::Process { process: Arc::new(process) }
+    pub fn extern_process(process_path: &Path) -> Result<ProcMacroClient, std::io::Error> {
+        let (thread, process) = ProcMacroProcessSrv::run(process_path)?;
+        Ok(ProcMacroClient {
+            kind: ProcMacroClientKind::Process { process: Arc::new(process), thread },
+        })
     }
 
     pub fn dummy() -> ProcMacroClient {
-        ProcMacroClient::Dummy
+        ProcMacroClient { kind: ProcMacroClientKind::Dummy }
     }
 
     pub fn by_dylib_path(
         &self,
-        _dylib_path: &Path,
+        dylib_path: &Path,
     ) -> Vec<(SmolStr, Arc<dyn ra_tt::TokenExpander>)> {
-        // FIXME: return empty for now
-        vec![]
+        match &self.kind {
+            ProcMacroClientKind::Dummy => vec![],
+            ProcMacroClientKind::Process { process, .. } => {
+                let macros = match process.find_proc_macros(dylib_path) {
+                    Err(err) => {
+                        eprintln!("Fail to find proc macro. Error: {:#?}", err);
+                        return vec![];
+                    }
+                    Ok(macros) => macros,
+                };
+
+                macros
+                    .into_iter()
+                    .filter_map(|(name, kind)| {
+                        // FIXME: Support custom derive only for now.
+                        match kind {
+                            ProcMacroKind::CustomDerive => {
+                                let name = SmolStr::new(&name);
+                                let expander: Arc<dyn ra_tt::TokenExpander> =
+                                    Arc::new(ProcMacroProcessExpander {
+                                        process: process.clone(),
+                                        name: name.clone(),
+                                        dylib_path: dylib_path.into(),
+                                    });
+                                Some((name, expander))
+                            }
+                            _ => None,
+                        }
+                    })
+                    .collect()
+            }
+        }
     }
 }

crates/ra_proc_macro/src/lib.rs

@@ -0,0 +1,93 @@
+//! Defines messages for cross-process message based on `ndjson` wire protocol
+
+use std::{
+    convert::TryFrom,
+    io::{self, BufRead, Write},
+};
+
+use crate::{
+    rpc::{ListMacrosResult, ListMacrosTask},
+    ExpansionResult, ExpansionTask,
+};
+use serde::{de::DeserializeOwned, Deserialize, Serialize};
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub enum Request {
+    ListMacro(ListMacrosTask),
+    ExpansionMacro(ExpansionTask),
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub enum Response {
+    Error(ResponseError),
+    ListMacro(ListMacrosResult),
+    ExpansionMacro(ExpansionResult),
+}
+
+macro_rules! impl_try_from_response {
+    ($ty:ty, $tag:ident) => {
+        impl TryFrom<Response> for $ty {
+            type Error = &'static str;
+            fn try_from(value: Response) -> Result<Self, Self::Error> {
+                match value {
+                    Response::$tag(res) => Ok(res),
+                    _ => Err("Fail to convert from response"),
+                }
+            }
+        }
+    };
+}
+
+impl_try_from_response!(ListMacrosResult, ListMacro);
+impl_try_from_response!(ExpansionResult, ExpansionMacro);
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct ResponseError {
+    pub code: ErrorCode,
+    pub message: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub enum ErrorCode {
+    ServerErrorEnd,
+    ExpansionError,
+}
+
+pub trait Message: Sized + Serialize + DeserializeOwned {
+    fn read(r: &mut impl BufRead) -> io::Result<Option<Self>> {
+        let text = match read_json(r)? {
+            None => return Ok(None),
+            Some(text) => text,
+        };
+        let msg = serde_json::from_str(&text)?;
+        Ok(Some(msg))
+    }
+    fn write(self, w: &mut impl Write) -> io::Result<()> {
+        let text = serde_json::to_string(&self)?;
+        write_json(w, &text)
+    }
+}
+
+impl Message for Request {}
+impl Message for Response {}
+
+fn read_json(inp: &mut impl BufRead) -> io::Result<Option<String>> {
+    let mut buf = String::new();
+    if inp.read_line(&mut buf)? == 0 {
+        return Ok(None);
+    }
+    // Remove ending '\n'
+    let buf = &buf[..buf.len() - 1];
+    if buf.is_empty() {
+        return Ok(None);
+    }
+    Ok(Some(buf.to_string()))
+}
+
+fn write_json(out: &mut impl Write, msg: &str) -> io::Result<()> {
+    log::debug!("> {}", msg);
+    out.write_all(msg.as_bytes())?;
+    out.write_all(b"\n")?;
+    out.flush()?;
+    Ok(())
+}