Allow multi-character symbols/variants (#92)

Author: T0mstone

build.rs

@@ -34,28 +34,28 @@ enum Def<'a> {
 
 /// A symbol, either a leaf or with modifiers with optional deprecation.
 enum Symbol<'a> {
-    Single(char),
-    Multi(Vec<(ModifierSet<&'a str>, char, Option<&'a str>)>),
+    Single(String),
+    Multi(Vec<(ModifierSet<&'a str>, String, Option<&'a str>)>),
 }
 
 /// A single line during parsing.
-#[derive(Debug, Copy, Clone)]
+#[derive(Debug, Clone)]
 enum Line<'a> {
     Blank,
     Deprecated(&'a str),
     ModuleStart(&'a str),
     ModuleEnd,
-    Symbol(&'a str, Option<char>),
-    Variant(ModifierSet<&'a str>, char),
+    Symbol(&'a str, Option<String>),
+    Variant(ModifierSet<&'a str>, String),
     Eof,
 }
 
-#[derive(Debug, Copy, Clone)]
+#[derive(Debug, Clone)]
 enum Declaration<'a> {
     ModuleStart(&'a str, Option<&'a str>),
     ModuleEnd,
-    Symbol(&'a str, Option<char>, Option<&'a str>),
-    Variant(ModifierSet<&'a str>, char, Option<&'a str>),
+    Symbol(&'a str, Option<String>, Option<&'a str>),
+    Variant(ModifierSet<&'a str>, String, Option<&'a str>),
 }
 
 fn main() {
@@ -103,11 +103,11 @@ fn process(buf: &mut String, file: &Path, name: &str, desc: &str) {
                     Some(Ok(Declaration::ModuleEnd))
                 }
             }
-            Ok(Line::Symbol(name, c)) => {
-                Some(Ok(Declaration::Symbol(name, c, deprecation.take())))
+            Ok(Line::Symbol(name, value)) => {
+                Some(Ok(Declaration::Symbol(name, value, deprecation.take())))
             }
-            Ok(Line::Variant(modifiers, c)) => {
-                Some(Ok(Declaration::Variant(modifiers, c, deprecation.take())))
+            Ok(Line::Variant(modifiers, value)) => {
+                Some(Ok(Declaration::Variant(modifiers, value, deprecation.take())))
             }
             Ok(Line::Eof) => {
                 deprecation.map(|_| Err(String::from("dangling `@deprecated:`")))
@@ -156,12 +156,12 @@ fn tokenize(line: &str) -> StrResult<Line> {
         for part in rest.split('.') {
             validate_ident(part)?;
         }
-        let c = decode_char(tail.ok_or("missing char")?)?;
-        Line::Variant(ModifierSet::from_raw_dotted(rest), c)
+        let value = decode_value(tail.ok_or("missing char")?)?;
+        Line::Variant(ModifierSet::from_raw_dotted(rest), value)
     } else {
         validate_ident(head)?;
-        let c = tail.map(decode_char).transpose()?;
-        Line::Symbol(head, c)
+        let value = tail.map(decode_value).transpose()?;
+        Line::Symbol(head, value)
     })
 }
 
@@ -174,20 +174,23 @@ fn validate_ident(string: &str) -> StrResult<()> {
     Err(format!("invalid identifier: {string:?}"))
 }
 
-/// Extracts either a single char or parses a U+XXXX escape.
-fn decode_char(text: &str) -> StrResult<char> {
-    if let Some(hex) = text.strip_prefix("U+") {
-        u32::from_str_radix(hex, 16)
-            .ok()
-            .and_then(|n| char::try_from(n).ok())
-            .ok_or_else(|| format!("invalid unicode escape {text:?}"))
-    } else {
-        let mut chars = text.chars();
-        match (chars.next(), chars.next()) {
-            (Some(c), None) => Ok(c),
-            _ => Err(format!("expected exactly one char, found {text:?}")),
-        }
+/// Extracts the value of a variant, parsing `\u{XXXX}` escapes
+fn decode_value(text: &str) -> StrResult<String> {
+    let mut iter = text.split("\\u{");
+    let mut res = iter.next().unwrap().to_string();
+    for other in iter {
+        let (hex, rest) = other.split_once("}").ok_or_else(|| {
+            format!("unclosed unicode escape \\u{{{}", other.escape_debug())
+        })?;
+        res.push(
+            u32::from_str_radix(hex, 16)
+                .ok()
+                .and_then(|n| char::try_from(n).ok())
+                .ok_or_else(|| format!("invalid unicode escape \\u{{{hex}}}"))?,
+        );
+        res += rest;
     }
+    Ok(res)
 }
 
 /// Turns a stream of lines into a list of definitions.
@@ -200,23 +203,23 @@ fn parse<'a>(
             None | Some(Declaration::ModuleEnd) => {
                 break;
             }
-            Some(Declaration::Symbol(name, c, deprecation)) => {
+            Some(Declaration::Symbol(name, value, deprecation)) => {
                 let mut variants = vec![];
-                while let Some(Declaration::Variant(name, c, deprecation)) =
+                while let Some(Declaration::Variant(name, value, deprecation)) =
                     p.peek().cloned().transpose()?
                 {
-                    variants.push((name, c, deprecation));
+                    variants.push((name, value, deprecation));
                     p.next();
                 }
 
                 let symbol = if !variants.is_empty() {
-                    if let Some(c) = c {
-                        variants.insert(0, (ModifierSet::default(), c, None));
+                    if let Some(value) = value {
+                        variants.insert(0, (ModifierSet::default(), value, None));
                     }
                     Symbol::Multi(variants)
                 } else {
-                    let c = c.ok_or("symbol needs char or variants")?;
-                    Symbol::Single(c)
+                    let value = value.ok_or("symbol needs char or variants")?;
+                    Symbol::Single(value)
                 };
 
                 defs.push((name, Binding { def: Def::Symbol(symbol), deprecation }));
@@ -251,7 +254,7 @@ fn encode(buf: &mut String, module: &Module) {
             Def::Symbol(symbol) => {
                 buf.push_str("Def::Symbol(Symbol::");
                 match symbol {
-                    Symbol::Single(c) => write!(buf, "Single({c:?})").unwrap(),
+                    Symbol::Single(value) => write!(buf, "Single({value:?})").unwrap(),
                     Symbol::Multi(list) => write!(buf, "Multi(&{list:?})").unwrap(),
                 }
                 buf.push(')');

src/lib.rs

@@ -62,15 +62,15 @@ pub enum Def {
 #[derive(Debug, Copy, Clone)]
 pub enum Symbol {
     /// A symbol without modifiers.
-    Single(char),
+    Single(&'static str),
     /// A symbol with named modifiers. The symbol defaults to its first variant.
-    Multi(&'static [(ModifierSet<&'static str>, char, Option<&'static str>)]),
+    Multi(&'static [(ModifierSet<&'static str>, &'static str, Option<&'static str>)]),
 }
 
 impl Symbol {
     /// Get the symbol's character for a given set of modifiers, alongside an optional deprecation
     /// message.
-    pub fn get(&self, modifs: ModifierSet<&str>) -> Option<(char, Option<&str>)> {
+    pub fn get(&self, modifs: ModifierSet<&str>) -> Option<(&'static str, Option<&str>)> {
         match self {
             Self::Single(c) => modifs.is_empty().then_some((*c, None)),
             Self::Multi(list) => {
@@ -84,13 +84,13 @@ impl Symbol {
     /// Each variant is represented by a tuple `(modifiers, character, deprecation)`.
     pub fn variants(
         &self,
-    ) -> impl Iterator<Item = (ModifierSet<&str>, char, Option<&str>)> {
+    ) -> impl Iterator<Item = (ModifierSet<&str>, &'static str, Option<&str>)> {
         enum Variants {
-            Single(std::iter::Once<char>),
+            Single(std::iter::Once<&'static str>),
             Multi(
                 std::slice::Iter<
                     'static,
-                    (ModifierSet<&'static str>, char, Option<&'static str>),
+                    (ModifierSet<&'static str>, &'static str, Option<&'static str>),
                 >,
             ),
         }
@@ -124,6 +124,7 @@ include!(concat!(env!("OUT_DIR"), "/out.rs"));
 #[cfg(test)]
 mod test {
     use super::*;
+    use std::collections::BTreeSet;
 
     #[test]
     fn all_modules_sorted() {
@@ -139,4 +140,48 @@ mod test {
 
         assert_sorted_recursively(ROOT);
     }
+
+    #[test]
+    fn unicode_escapes() {
+        let Def::Symbol(wj) = SYM.get("wj").unwrap().def else { panic!() };
+        assert_eq!(wj.get(ModifierSet::default()).unwrap().0, "\u{2060}");
+        let Def::Symbol(space) = SYM.get("space").unwrap().def else { panic!() };
+        assert_eq!(space.get(ModifierSet::default()).unwrap().0, " ");
+        assert_eq!(
+            space.get(ModifierSet::from_raw_dotted("nobreak")).unwrap().0,
+            "\u{A0}"
+        );
+    }
+
+    #[test]
+    fn random_sample() {
+        for (key, control) in [
+            ("backslash", [("", "\\"), ("circle", "⦸"), ("not", "⧷")].as_slice()),
+            ("chi", &[("", "χ")]),
+            ("forces", &[("", "⊩"), ("not", "⊮")]),
+            ("interleave", &[("", "⫴"), ("big", "⫼"), ("struck", "⫵")]),
+            ("uranus", &[("", "⛢"), ("alt", "♅")]),
+        ] {
+            let Def::Symbol(s) = SYM.get(key).unwrap().def else {
+                panic!("{key:?} is not a symbol")
+            };
+            let variants = s
+                .variants()
+                .map(|(m, v, _)| (m.into_iter().collect::<BTreeSet<_>>(), v))
+                .collect::<BTreeSet<_>>();
+            let control = control
+                .iter()
+                .map(|&(m, v)| {
+                    (
+                        ModifierSet::from_raw_dotted(m)
+                            .into_iter()
+                            .collect::<BTreeSet<_>>(),
+                        v,
+                    )
+                })
+                .collect::<BTreeSet<_>>();
+
+            assert_eq!(variants, control);
+        }
+    }
 }

src/modules/sym.txt

@@ -1,25 +1,25 @@
 // Control.
-wj U+2060
-zwj U+200D
-zwnj U+200C
-zws U+200B
-lrm U+200E
-rlm U+200F
+wj \u{2060}
+zwj \u{200D}
+zwnj \u{200C}
+zws \u{200B}
+lrm \u{200E}
+rlm \u{200F}
 
 // Spaces.
-space U+20
-  .nobreak U+A0
-  .nobreak.narrow U+202F
-  .en U+2002
-  .quad U+2003
-  .third U+2004
-  .quarter U+2005
-  .sixth U+2006
-  .med U+205F
-  .fig U+2007
-  .punct U+2008
-  .thin U+2009
-  .hair U+200A
+space \u{20}
+  .nobreak \u{A0}
+  .nobreak.narrow \u{202F}
+  .en \u{2002}
+  .quad \u{2003}
+  .third \u{2004}
+  .quarter \u{2005}
+  .sixth \u{2006}
+  .med \u{205F}
+  .fig \u{2007}
+  .punct \u{2008}
+  .thin \u{2009}
+  .hair \u{200A}
 
 // Delimiters.
 paren
@@ -30,9 +30,9 @@ paren
   .t ⏜
   .b ⏝
 brace
-  .l U+7B
+  .l \u{7B}
   .l.double ⦃
-  .r U+7D
+  .r \u{7D}
   .r.double ⦄
   .t ⏞
   .b ⏟
@@ -141,14 +141,14 @@ dash
   .wave.double 〰
 dot
   .op ⋅
-  .basic U+2E
+  .basic \u{2E}
   .c ·
   .circle ⊙
   .circle.big ⨀
   .square ⊡
   .double ¨
-  .triple U+20DB
-  .quad U+20DC
+  .triple \u{20DB}
+  .quad \u{20DC}
 excl !
   .double ‼
   .inv ¡
@@ -161,10 +161,10 @@ interrobang ‽
   .inv ⸘
 hash #
 hyph ‐
-  .minus U+2D
-  .nobreak U+2011
+  .minus \u{2D}
+  .nobreak \u{2011}
   .point ‧
-  .soft U+AD
+  .soft \u{AD}
 numero №
 percent %
 permille ‰