improves how \ is handled in the prescanner (#49)

* improves how \ is handled in the prescanner

* fixup: Format Python code with Black

* add a comment. simplifying the logic.

* comment rewording by Rocky

* adding length tests

Co-authored-by: autoblack <[email protected]>

Author: mmatera

mathics_scanner/prescanner.py

@@ -59,7 +59,12 @@ def scan(self):
                     self.stubs.append(self.code[self.start : self.pos])
                     self.newstub(self.pos + 2)
                 else:
-                    self.pos += 1
+                    # Two backslashes in succession indicates a single backslash character,
+                    # rather than an escape sequence which also starts with a backslash.
+                    #  Advance the scanning cursor (self.pos) over both backslashes.
+                    # Also, Python's backslash escape mechanism turns the two backslashes
+                    # into one in length calculations.
+                    self.pos += 2
             else:
                 self.pos += 1
         self.stubs.append(self.code[self.start :])  # final stub

test/test_prescanner.py

@@ -21,11 +21,25 @@ def incomplete(self, code):
     def equal(self, code, result):
         assert self.prescan(code) == result
 
+    def equal_length(self, code, length):
+        assert len(self.prescan(code)) == length
+
     def test_longnames(self):
         self.equal(r"\[Theta]", "\u03B8")
         self.equal(r"\[CapitalPi]", "\u03A0")
         self.equal(r"\[Fake]", r"\[Fake]")
         self.equal("z \\[Conjugate]", "z \uF3C8")
+        self.equal("z \\[Integral]", "z \u222b")
+        self.equal("z \\\\[Integral]", "z \\\\[Integral]")
+        self.equal("z \\\\\\[Integral]", "z \\\\\u222b")
+        self.equal("abc\\\\", "abc\\\\")
+
+    def test_lengths(self):
+        self.equal_length(r'"\[Integral]"', 3)
+        # Prescanner keep both slashes and quotes.
+        # The tokenizer brings \\ into \ if it appears
+        # inside a string.
+        self.equal_length(r'"\\[Integral]"', 14)
 
     def test_oct(self):
         self.equal(r"\051", ")")