internal: more production-ready proc-macro RPC deserialization

* avoid arbitrary nested JSON tree (danger of stack overflow)
* use more compact representation.

Author: matklad

crates/proc_macro_api/src/lib.rs

@@ -21,7 +21,9 @@ use tt::{SmolStr, Subtree};
 
 use crate::process::ProcMacroProcessSrv;
 
-pub use rpc::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTask, ProcMacroKind};
+pub use rpc::{
+    ExpansionResult, ExpansionTask, flat::FlatTree, ListMacrosResult, ListMacrosTask, ProcMacroKind,
+};
 pub use version::{read_dylib_info, RustCInfo};
 
 #[derive(Debug, Clone)]
@@ -58,9 +60,9 @@ impl ProcMacroProcessExpander {
         env: Vec<(String, String)>,
     ) -> Result<Subtree, tt::ExpansionError> {
         let task = ExpansionTask {
-            macro_body: subtree.clone(),
+            macro_body: FlatTree::new(subtree),
             macro_name: self.name.to_string(),
-            attributes: attr.cloned(),
+            attributes: attr.map(FlatTree::new),
             lib: self.dylib_path.to_path_buf(),
             env,
         };
@@ -70,7 +72,7 @@ impl ProcMacroProcessExpander {
             .lock()
             .unwrap_or_else(|e| e.into_inner())
             .send_task(msg::Request::ExpansionMacro(task))?;
-        Ok(result.expansion)
+        Ok(result.expansion.to_subtree())
     }
 }
 

crates/proc_macro_api/src/msg.rs

@@ -12,13 +12,13 @@ use crate::{
     ExpansionResult, ExpansionTask,
 };
 
-#[derive(Debug, Serialize, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize)]
 pub enum Request {
     ListMacro(ListMacrosTask),
     ExpansionMacro(ExpansionTask),
 }
 
-#[derive(Debug, Serialize, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize)]
 pub enum Response {
     Error(ResponseError),
     ListMacro(ListMacrosResult),

crates/proc_macro_api/src/rpc.rs

@@ -5,13 +5,12 @@
 //! Although adding `Serialize` and `Deserialize` traits to `tt` directly seems
 //! to be much easier, we deliberately duplicate `tt` structs with `#[serde(with = "XXDef")]`
 //! for separation of code responsibility.
+pub(crate) mod flat;
 
 use paths::AbsPathBuf;
 use serde::{Deserialize, Serialize};
-use tt::{
-    Delimiter, DelimiterKind, Ident, Leaf, Literal, Punct, SmolStr, Spacing, Subtree, TokenId,
-    TokenTree,
-};
+
+use crate::rpc::flat::FlatTree;
 
 #[derive(Clone, Eq, PartialEq, Debug, Serialize, Deserialize)]
 pub struct ListMacrosTask {
@@ -30,14 +29,13 @@ pub struct ListMacrosResult {
     pub macros: Vec<(String, ProcMacroKind)>,
 }
 
-#[derive(Clone, Eq, PartialEq, Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct ExpansionTask {
     /// Argument of macro call.
     ///
     /// In custom derive this will be a struct or enum; in attribute-like macro - underlying
     /// item; in function-like macro - the macro body.
-    #[serde(with = "SubtreeDef")]
-    pub macro_body: Subtree,
+    pub macro_body: FlatTree,
 
     /// Name of macro to expand.
     ///
@@ -46,208 +44,23 @@ pub struct ExpansionTask {
     pub macro_name: String,
 
     /// Possible attributes for the attribute-like macros.
-    #[serde(with = "opt_subtree_def")]
-    pub attributes: Option<Subtree>,
+    pub attributes: Option<FlatTree>,
 
     pub lib: AbsPathBuf,
 
     /// Environment variables to set during macro expansion.
     pub env: Vec<(String, String)>,
 }
 
-#[derive(Clone, Eq, PartialEq, Debug, Default, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct ExpansionResult {
-    #[serde(with = "SubtreeDef")]
-    pub expansion: Subtree,
-}
-
-#[derive(Serialize, Deserialize)]
-#[serde(remote = "DelimiterKind")]
-enum DelimiterKindDef {
-    Parenthesis,
-    Brace,
-    Bracket,
-}
-
-#[derive(Serialize, Deserialize)]
-#[serde(remote = "TokenId")]
-struct TokenIdDef(u32);
-
-#[derive(Serialize, Deserialize)]
-#[serde(remote = "Delimiter")]
-struct DelimiterDef {
-    #[serde(
-        with = "TokenIdDef",
-        default = "tt::TokenId::unspecified",
-        skip_serializing_if = "token_id_def::skip_if"
-    )]
-    id: TokenId,
-    #[serde(with = "DelimiterKindDef")]
-    kind: DelimiterKind,
-}
-
-#[derive(Serialize, Deserialize)]
-#[serde(remote = "Subtree")]
-struct SubtreeDef {
-    #[serde(default, with = "opt_delimiter_def")]
-    delimiter: Option<Delimiter>,
-    #[serde(with = "vec_token_tree")]
-    token_trees: Vec<TokenTree>,
-}
-
-#[derive(Serialize, Deserialize)]
-#[serde(remote = "TokenTree")]
-enum TokenTreeDef {
-    #[serde(with = "LeafDef")]
-    Leaf(Leaf),
-    #[serde(with = "SubtreeDef")]
-    Subtree(Subtree),
-}
-
-#[derive(Serialize, Deserialize)]
-#[serde(remote = "Leaf")]
-enum LeafDef {
-    #[serde(with = "LiteralDef")]
-    Literal(Literal),
-    #[serde(with = "PunctDef")]
-    Punct(Punct),
-    #[serde(with = "IdentDef")]
-    Ident(Ident),
-}
-
-#[derive(Serialize, Deserialize)]
-#[serde(remote = "Literal")]
-struct LiteralDef {
-    text: SmolStr,
-    #[serde(
-        with = "TokenIdDef",
-        default = "tt::TokenId::unspecified",
-        skip_serializing_if = "token_id_def::skip_if"
-    )]
-    id: TokenId,
-}
-
-#[derive(Serialize, Deserialize)]
-#[serde(remote = "Punct")]
-struct PunctDef {
-    char: char,
-    #[serde(with = "SpacingDef")]
-    spacing: Spacing,
-    #[serde(
-        with = "TokenIdDef",
-        default = "tt::TokenId::unspecified",
-        skip_serializing_if = "token_id_def::skip_if"
-    )]
-    id: TokenId,
-}
-
-#[derive(Serialize, Deserialize)]
-#[serde(remote = "Spacing")]
-enum SpacingDef {
-    Alone,
-    Joint,
-}
-
-#[derive(Serialize, Deserialize)]
-#[serde(remote = "Ident")]
-struct IdentDef {
-    text: SmolStr,
-    #[serde(
-        with = "TokenIdDef",
-        default = "tt::TokenId::unspecified",
-        skip_serializing_if = "token_id_def::skip_if"
-    )]
-    id: TokenId,
-}
-
-mod token_id_def {
-    pub(super) fn skip_if(value: &tt::TokenId) -> bool {
-        *value == tt::TokenId::unspecified()
-    }
-}
-
-mod opt_delimiter_def {
-    use super::{Delimiter, DelimiterDef};
-    use serde::{Deserialize, Deserializer, Serialize, Serializer};
-
-    pub(super) fn serialize<S>(value: &Option<Delimiter>, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        #[derive(Serialize)]
-        struct Helper<'a>(#[serde(with = "DelimiterDef")] &'a Delimiter);
-        value.as_ref().map(Helper).serialize(serializer)
-    }
-
-    pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<Option<Delimiter>, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        #[derive(Deserialize)]
-        struct Helper(#[serde(with = "DelimiterDef")] Delimiter);
-        let helper = Option::deserialize(deserializer)?;
-        Ok(helper.map(|Helper(external)| external))
-    }
-}
-
-mod opt_subtree_def {
-    use super::{Subtree, SubtreeDef};
-    use serde::{Deserialize, Deserializer, Serialize, Serializer};
-
-    pub(super) fn serialize<S>(value: &Option<Subtree>, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        #[derive(Serialize)]
-        struct Helper<'a>(#[serde(with = "SubtreeDef")] &'a Subtree);
-        value.as_ref().map(Helper).serialize(serializer)
-    }
-
-    pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<Option<Subtree>, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        #[derive(Deserialize)]
-        struct Helper(#[serde(with = "SubtreeDef")] Subtree);
-        let helper = Option::deserialize(deserializer)?;
-        Ok(helper.map(|Helper(external)| external))
-    }
-}
-
-mod vec_token_tree {
-    use super::{TokenTree, TokenTreeDef};
-    use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize, Serializer};
-
-    pub(super) fn serialize<S>(value: &[TokenTree], serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        #[derive(Serialize)]
-        struct Helper<'a>(#[serde(with = "TokenTreeDef")] &'a TokenTree);
-
-        let items: Vec<_> = value.iter().map(Helper).collect();
-        let mut seq = serializer.serialize_seq(Some(items.len()))?;
-        for element in items {
-            seq.serialize_element(&element)?;
-        }
-        seq.end()
-    }
-
-    pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<Vec<TokenTree>, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        #[derive(Deserialize)]
-        struct Helper(#[serde(with = "TokenTreeDef")] TokenTree);
-
-        let helper = Vec::deserialize(deserializer)?;
-        Ok(helper.into_iter().map(|Helper(external)| external).collect())
-    }
+    pub expansion: FlatTree,
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use tt::*;
 
     fn fixture_token_tree() -> Subtree {
         let mut subtree = Subtree::default();
@@ -257,6 +70,15 @@ mod tests {
         subtree
             .token_trees
             .push(TokenTree::Leaf(Ident { text: "Foo".into(), id: TokenId(1) }.into()));
+        subtree.token_trees.push(TokenTree::Leaf(Leaf::Literal(Literal {
+            text: "Foo".into(),
+            id: TokenId::unspecified(),
+        })));
+        subtree.token_trees.push(TokenTree::Leaf(Leaf::Punct(Punct {
+            char: '@',
+            id: TokenId::unspecified(),
+            spacing: Spacing::Joint,
+        })));
         subtree.token_trees.push(TokenTree::Subtree(Subtree {
             delimiter: Some(Delimiter { id: TokenId(2), kind: DelimiterKind::Brace }),
             token_trees: vec![],
@@ -268,22 +90,17 @@ mod tests {
     fn test_proc_macro_rpc_works() {
         let tt = fixture_token_tree();
         let task = ExpansionTask {
-            macro_body: tt.clone(),
+            macro_body: FlatTree::new(&tt),
             macro_name: Default::default(),
             attributes: None,
             lib: AbsPathBuf::assert(std::env::current_dir().unwrap()),
             env: Default::default(),
         };
 
         let json = serde_json::to_string(&task).unwrap();
+        println!("{}", json);
         let back: ExpansionTask = serde_json::from_str(&json).unwrap();
 
-        assert_eq!(task.macro_body, back.macro_body);
-
-        let result = ExpansionResult { expansion: tt };
-        let json = serde_json::to_string(&result).unwrap();
-        let back: ExpansionResult = serde_json::from_str(&json).unwrap();
-
-        assert_eq!(result, back);
+        assert_eq!(tt, back.macro_body.to_subtree());
     }
 }