Disable P8SCII unescaping to fix mangling of emoji characters

pico8/game/formatter/p8.py

@@ -308,7 +308,7 @@ def to_file(
         for line in game.lua.to_lines(
                 writer_cls=lua_writer_cls,
                 writer_args=lua_writer_args):
-            outstr.write(bytes(lua.p8scii_to_unicode(line), 'utf-8'))
+            outstr.write(line)
             ended_in_newline = line.endswith(b'\n')
         if not ended_in_newline:
             outstr.write(b'\n')

pico8/lua/lexer.py

@@ -161,9 +161,7 @@ def code(self):
             escaped_chrs = []
             for c in self._data:
                 c = bytes([c])
-                if c in _STRING_REVERSE_ESCAPES:
-                    escaped_chrs.append(b'\\' + _STRING_REVERSE_ESCAPES[c])
-                elif c == self._quote:
+                if c == self._quote:
                     escaped_chrs.append(b'\\' + c)
                 else:
                     escaped_chrs.append(c)
@@ -376,18 +374,6 @@ def _process_token(self, s):
                     i += 1
                     break
 
-                if c == b'\\':
-                    # Escape character.
-                    num_m = re.match(br'\d{1,3}', s[i+1:])
-                    if num_m:
-                        c = bytes([int(num_m.group(0))])
-                        i += len(num_m.group(0))
-                    else:
-                        next_c = s[i+1:i+2]
-                        if next_c in _STRING_ESCAPES:
-                            c = _STRING_ESCAPES[next_c]
-                            i += 1
-
                 self._in_string.append(c)
                 i += 1
 

tests/pico8/game/formatter/formatter_test.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+import unittest
+
+from pico8.game.formatter.p8 import P8Formatter
+from pico8.game import game
+from pico8.lua import lua
+from io import BytesIO
+
+class TestP8Formatter(unittest.TestCase):
+	def testEmojisArePreservedThroughFormatting(self):
+		lua_code = 'print(\"Hello Emoji 🅾️\")'
+		lua_code_utf8_bytes = lua_code.encode('utf-8')
+		emoji_bytes = '🅾️'.encode('utf-8')
+
+		output_cart = game.Game.make_empty_game('dummy.p8')
+		output_cart.lua = lua.Lua.from_lines([lua_code_utf8_bytes], version=8)
+
+		cart_stream = BytesIO()
+
+		formatter = P8Formatter()
+		formatter.to_file(
+			output_cart,
+			cart_stream,
+			lua_writer_cls = lua.LuaEchoWriter)
+
+		cart_bytes = cart_stream.getvalue()
+
+		assert(emoji_bytes in cart_bytes)

tests/pico8/lua/lexer_test.py

@@ -257,13 +257,6 @@ def testStringMultipleLinesPlusAToken(self):
                          lxr._tokens[0])
         self.assertEqual(lexer.TokKeyword(b'and'), lxr._tokens[2])
 
-    def testStringEscapes(self):
-        lxr = lexer.Lexer(version=4)
-        lxr._process_line(b'"\\\n\\a\\b\\f\\n\\r\\t\\v\\\\\\"\\\'\\65"\n')
-        self.assertEqual(2, len(lxr._tokens))
-        self.assertEqual(lexer.TokString(b'\n\a\b\f\n\r\t\v\\"\'A'),
-                         lxr._tokens[0])
-
     def testComment(self):
         lxr = lexer.Lexer(version=4)
         lxr._process_line(b'-- comment text and stuff\n')