support for 32 bits unicode characters (#48)

Author: mmatera

mathics_scanner/prescanner.py

@@ -10,6 +10,7 @@ class Prescanner(object):
         character codes to characters:
             \.7A -> z
             \:004a -> J
+            \|01D451  -> \U0001D451
             \041 -> !
         unicode longnames to characters:
             \[Theta] -> \u03B8
@@ -45,6 +46,8 @@ def scan(self):
                 if self.pos + 1 == len(self.code):
                     self.incomplete()
                 c = self.code[self.pos + 1]
+                if c == "|":
+                    self.try_parse_base(2, 8, 16)
                 if c == ".":
                     self.try_parse_base(2, 4, 16)
                 elif c == ":":

test/test_prescanner.py

@@ -52,6 +52,9 @@ def test_hex_colon(self):
         self.equal(r"\:03B8", "\u03B8")
         self.equal(r"\:03b8", "\u03B8")
 
+    def test_hex_vbar(self):
+        self.equal(r"\|01D451", "\U0001D451")
+
     def test_incomplete(self):
         self.incomplete(r"\[")
         self.incomplete(r"\[Theta")