Merge pull request #139 from Distributive-Network/Xmader/fix/readline-windows

Make pmjs & peter-jr work on Windows

Author: zollqir

.github/workflows/test-and-publish.yaml

@@ -159,8 +159,6 @@ jobs:
           poetry run python -m pip install --force-reinstall --verbose ./dist/*
           poetry run python -m pytest tests/python
       - name: Run JS tests (peter-jr)
-        if: ${{ runner.os != 'Windows' }} # Python on Windows doesn't have the readline library
-        # FIXME: on macOS we must make sure to use the GNU version of wc and realpath
         run: |
           poetry run bash ./peter-jr ./tests/js/
   sdist:

poetry.lock

@@ -28,6 +28,7 @@ include = [
 
 [tool.poetry.dependencies]
 python = "^3.8"
+pyreadline3 = "^3.4.1"
 pminit = { version = "*", allow-prereleases = true }
 
 

pyproject.toml

@@ -3,13 +3,17 @@
 # @author       Wes Garland, [email protected]
 # @date         June 2023
 
-import sys, os, readline, signal, getopt
+import sys, os, signal, getopt
+try:
+  import readline # Unix
+except ImportError:
+  import pyreadline3 as readline # Windows
 import pythonmonkey as pm
 globalThis = pm.eval("globalThis")
 evalOpts = { 'filename': __file__, 'fromPythonFrame': True, 'strict': False } # type: pm.EvalOptions
 
 if (os.getenv('PMJS_PATH')):
-    requirePath = list(map(os.path.abspath, os.getenv('PMJS_PATH').split(':')))
+    requirePath = list(map(os.path.abspath, os.getenv('PMJS_PATH').split(',')))
 else:
     requirePath = False;
 
@@ -108,7 +112,7 @@
    * like that which is also a valid compilation unit with parens, then if that is a syntax error, 
    * we re-evaluate without the parens.
    */
-  if (/^\\s*\{.*[^;\\s]\\s*$/.test(statement))
+  if (/^\\s*\\{.*[^;\\s]\\s*$/.test(statement))
   {
     const testStatement = `(${statement})`;
     if (globalThis.python.pythonMonkey.isCompilableUnit(testStatement))
@@ -195,8 +199,7 @@ def sigint_handler(signum, frame):
 
         got_sigint = got_sigint + 1
         if (got_sigint > 1):
-            sys.stdout.write("\n")
-            quit()
+            raise EOFError
 
         if (inner_loop != True):
             if (got_sigint == 1 and len(readline.get_line_buffer()) == readline_skip_chars):
@@ -304,7 +307,7 @@ def initGlobalThis():
 
     require = pm.createRequire(os.path.abspath(os.getcwd() + '/__pmjs_virtual__'), requirePath)
     globalThis.require = require
-    globalInitModule = require(os.path.dirname(__file__) + "/../lib/pmjs/global-init") # module load has side-effects
+    globalInitModule = require(os.path.realpath(os.path.dirname(__file__) + "/../lib/pmjs/global-init")) # module load has side-effects
     argvBuilder = globalInitModule.makeArgvBuilder()
     for arg in sys.argv:
         argvBuilder(arg); # list=>Array not working yet

python/pythonmonkey/cli/pmjs.py

@@ -23,7 +23,7 @@
 # @date         May 2023
 #
 
-import sys, os
+import sys, os, io
 from typing import Union, Dict, Literal, List
 import importlib
 import importlib.util
@@ -43,6 +43,12 @@
 )
 evalOpts = { 'filename': __file__, 'fromPythonFrame': True } # type: pm.EvalOptions
 
+# Force to use UTF-8 encoding
+# Windows may use other encodings / code pages that have many characters missing/unrepresentable
+# Error: Python UnicodeEncodeError: 'charmap' codec can't encode characters in position xx-xx: character maps to <undefined>
+sys.stdout.reconfigure(encoding='utf-8')
+sys.stderr.reconfigure(encoding='utf-8')
+
 # Add some python functions to the global python object for code in this file to use.
 globalThis = pm.eval("globalThis;", evalOpts)
 pm.eval("globalThis.python = { pythonMonkey: {}, stdout: {}, stderr: {} }", evalOpts);
@@ -60,8 +66,7 @@
 globalThis.python.eval = eval
 globalThis.python.exec = exec
 globalThis.python.getenv = os.getenv
-globalThis.python.paths  = ':'.join(sys.path)
-pm.eval("python.paths = python.paths.split(':');", evalOpts); # fix when pm supports arrays
+globalThis.python.paths  = sys.path
 
 globalThis.python.exit = pm.eval("""'use strict';
 (exit) => function pythonExitWrapper(exitCode) {
@@ -269,6 +274,7 @@ def _createRequireInner(*args):
  */
 function createRequire(filename, bootstrap_broken, extraPaths, isMain)
 {
+  filename = filename.split('\\\\').join('/');
   const bootstrap = globalThis.bootstrap; /** @bug PM-65 */
   const CtxModule = bootstrap.modules['ctx-module'].CtxModule;
   const moduleCache = globalThis.require?.cache || {};
@@ -283,7 +289,7 @@ def _createRequireInner(*args):
 
   const module = new CtxModule(globalThis, filename, moduleCache);
   moduleCache[filename] = module;
-  for (let path of python.paths)
+  for (let path of Array.from(python.paths))
     module.paths.push(path + '/node_modules');
   module.require.path.push(python.pythonMonkey.dir + '/builtin_modules');
   module.require.path.push(python.pythonMonkey.nodeModules);
@@ -300,7 +306,7 @@ def _createRequireInner(*args):
   }
 
   if (extraPaths)
-    module.require.path.splice(module.require.path.length, 0, ...(extraPaths.split(':')));
+    module.require.path.splice(module.require.path.length, 0, ...(extraPaths.split(',')));
 
   return module.require;
 })""", evalOpts)(*args)

python/pythonmonkey/require.py

@@ -20,6 +20,7 @@ runTest()
   echo -n "${testName}: "
 
   PMJS_PATH="`pwd`" pmjs -e 'print=python.print' program.js\
+  | tr -d '\r'\
   | while read word rest
     do
       case "$word" in

tests/js/commonjs-modules.bash

@@ -22,6 +22,7 @@ cd `dirname "$0"` || panic "could not change to test directory"
 -e 'console.debug("stdout")' \
 -e 'console.info("stdout")' \
 < /dev/null \
+| tr -d '\r' \
 | grep -c '^stdout$' \
 | while read qty
   do
@@ -34,6 +35,7 @@ cd `dirname "$0"` || panic "could not change to test directory"
 -e 'console.error("stderr")' \
 -e 'console.warn("stderr")' \
 < /dev/null 2>&1 \
+| tr -d '\r' \
 | grep -c '^stderr$' \
 | while read qty
   do

tests/js/console-stdio.bash

@@ -18,6 +18,7 @@ panic()
 cd `dirname "$0"` || panic "could not change to test directory"
 
 "${PMJS:-pmjs}" -e 'console.log("OKAY")' < /dev/null |\
+tr -d '\r' |\
 while read keyword rest
 do
   case "$keyword" in

tests/js/pmjs-eopt.bash

@@ -20,6 +20,7 @@ cd `dirname "$0"` || panic "could not change to test directory"
 
 argc=0
 "${PMJS:-pmjs}" program.js abc easy as one two three |\
+tr -d '\r' |\
 while read keyword rest
 do
   case "$keyword" in

tests/js/pmjs-global-arguments.bash

@@ -30,7 +30,7 @@ a;
 b(${rnd});
 EOF
 )\
-| cat -u | while read prompt keyword rest
+| cat -u | tr -d '\r' | while read prompt keyword rest
   do
     case "$keyword" in
       "...")