Merge branch 'main' into Xmader/feat/pm.require

Author: Xmader

.github/workflows/test-and-publish.yaml

@@ -24,9 +24,9 @@ jobs:
   build-spidermonkey-unix:
     strategy:
       matrix:
-        # Use Ubuntu 20.04 / macOS 12 + Python 3.8 to build SpiderMonkey
-        os: [ 'ubuntu-20.04', 'macos-12' ]
-        python_version: [ '3.8' ]
+        # Use Ubuntu 20.04 / macOS 12 + Python 3.10 to build SpiderMonkey
+        os: [ 'ubuntu-20.04', 'macos-12', 'm2ci' ]
+        python_version: [ '3.10' ]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
@@ -39,7 +39,7 @@ jobs:
         with:
           path: |
             ./_spidermonkey_install/*
-          key: ${{ runner.os }}-spidermonkey
+          key: spidermonkey-${{ runner.os }}-${{ runner.arch }}
           lookup-only: true # skip download
       - name: Build spidermonkey
         if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
@@ -54,7 +54,7 @@ jobs:
         with:
           path: |
             ./_spidermonkey_install/*
-          key: ${{ runner.os }}-spidermonkey
+          key: spidermonkey-${{ runner.os }}-${{ runner.arch }}
           lookup-only: true # skip download
       - name: Install dependencies
         if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
@@ -78,41 +78,61 @@ jobs:
       fail-fast: false
       matrix:
         # The lowest supported version is Ubuntu 20.04 + Python 3.8 or macOS 12 + Python 3.9
-        os: [ 'ubuntu-20.04', 'macos-12', 'windows-2019' ]
+        os: [ 'ubuntu-20.04', 'macos-12', 'windows-2019', 'm2ci' ]
         python_version: [ '3.8', '3.9', '3.10', '3.11', '3.12-dev' ]
         exclude:
           # macOS 12 comes with Python 3.9 by default, so we drop ci support for Python 3.8 on macOS
           # FIXME: We can't build on macOS 11 for now because our prebuilt `uncrustify` binary requires macOS 12
           - os: 'macos-12'
             python_version: '3.8'
+          # actions/setup-python: The version '3.8'/'3.9' with architecture 'arm64' was not found for macOS.
+          # see https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json
+          - os: 'm2ci'
+            python_version: '3.8'
+          - os: 'm2ci'
+            python_version: '3.9'
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # fetch all history for all branches and tags
+          # poetry-dynamic-versioning needs git tags to produce the correct version number 
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python_version }}
       - name: Setup Poetry
         uses: snok/install-poetry@v1
+        with:
+          version: 1.5.1
       - name: Install Dependencies
         run: |
           echo "Installing Dependencies"
           if [[ "$OSTYPE" == "linux-gnu"* ]]; then # Linux
             sudo apt-get update -y
             sudo apt-get install -y cmake doxygen graphviz llvm
           elif [[ "$OSTYPE" == "darwin"* ]]; then # macOS
-            brew update
+            brew update || true # allow failure
             brew install cmake doxygen graphviz pkg-config wget coreutils # `coreutils` installs the `realpath` command
           fi
           echo "Installing python deps"
-          poetry install --no-root
+          poetry self add "poetry-dynamic-versioning[plugin]"
+          poetry env use python3 # use the correct Python version we've set up
+          poetry install --no-root --only=dev
           echo "Installed Dependencies"
       - name: Use cached spidermonkey build
         uses: actions/cache@v3
         with:
           path: |
             ./_spidermonkey_install/*
-          key: ${{ runner.os }}-spidermonkey
+          key: spidermonkey-${{ runner.os }}-${{ runner.arch }}
           fail-on-cache-miss: true # SpiderMonkey is expected to be cached in its dedicated job
+      - name: Build pminit
+        run: |
+          cd python/pminit
+          poetry build --format=sdist
+          cd -
+          mkdir -p ./dist/
+          mv -v python/pminit/dist/* ./dist/
       - name: Build wheel
         run: |
           echo $(poetry run python --version)
@@ -125,19 +145,24 @@ jobs:
           path: ./dist/
       - name: Run Python tests (pytest)
         run: |
-          poetry run python -m pip install ./dist/*.whl
+          poetry run python -m pip install --force-reinstall ./dist/*
           poetry run python -m pytest tests/python
   sdist:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
       - uses: actions/setup-python@v4
         with:
           python-version: '3.9'
       - name: Setup Poetry
         uses: snok/install-poetry@v1
+        with:
+          version: 1.5.1
       - name: Build source distribution (sdist) file
         run: |
+          poetry self add "poetry-dynamic-versioning[plugin]"
           poetry build --format=sdist
           ls -lah ./dist/
       - name: Upload sdist as CI artifacts
@@ -156,6 +181,8 @@ jobs:
           python-version: '3.9'
       - name: Setup Poetry
         uses: snok/install-poetry@v1
+        with:
+          version: 1.5.1
       - name: Download wheels built
         uses: actions/download-artifact@v3
         with:
@@ -185,16 +212,26 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: wheel-${{ github.run_id }}-${{ github.sha }}
-          path: ./pythonmonkey/ # write to the repository project path 
+          path: ./dist/
+      - name: Move wheels to the correct repository project paths
+        run: |
+          mkdir -p ./pythonmonkey/ ./pminit/
+          mv ./dist/pythonmonkey-* ./pythonmonkey/
+          mv ./dist/pminit-* ./pminit/
       - name: Generate index page for the project
         run: |
-          cd ./pythonmonkey/
-          html="<html><head><title>PythonMonkey Nightly Builds</title></head><body>"
-          for file in ./*; do # generate <a> tags for each file
-            html+="<a href=\"$file\">$file</a><br/>"
-          done
-          html+="</body></html>"
-          echo "$html" > ./index.html
+          generate_index () {
+            cd $1
+            html="<html><head><title>PythonMonkey Nightly Builds</title></head><body>"
+            for file in ./*; do # generate <a> tags for each file
+              html+="<a href=\"$file\">$file</a><br/>"
+            done
+            html+="</body></html>"
+            echo "$html" > ./index.html
+            cd -
+          }
+          generate_index ./pythonmonkey/
+          generate_index ./pminit/
       - name: Generate repository root page
         run: |
           html="<html><head><title>PythonMonkey Nightly Builds</title></head><body>"
@@ -203,6 +240,7 @@ jobs:
           html+="<pre>pip install -i https://nightly.pythonmonkey.io/ --pre pythonmonkey</pre>"
           html+="<h3>Browse files:</h3>"
           html+="<a href="pythonmonkey/">pythonmonkey</a>"
+          html+="<a href="pminit/">pminit</a>"
           html+="</body></html>"
           echo "$html" > ./index.html
       - uses: actions/upload-pages-artifact@v1

.gitignore

@@ -1,7 +1,6 @@
 build/
 .vscode/
 pyvenv.cfg
-python/pythonmonkey/version.py
 python/pythonmonkey/node_modules
 bin/
 lib/*
@@ -15,12 +14,6 @@ _spidermonkey_install
 __pycache__/*
 dist
 *.so
-_spidermonkey_install/*
-*~
-Makefile
-CMakeDoxyfile.in
-CMakeDoxygenDefaults.cmake
-CMakeCache.txt
 *.dylib
 *.dll
 *.pyd

CMakeLists.txt

@@ -1,6 +1,6 @@
 # Copyright (c) 2022 Distributive Inc. All Rights Reserved.
 
-cmake_minimum_required(VERSION 3.4...3.18) # Set minimum cmake version
+cmake_minimum_required(VERSION 3.25) # Set minimum cmake version
 
 project("PythonMonkey"
   DESCRIPTION "A tool for Javascript-Python interoperability."

README.md

@@ -6,7 +6,7 @@
 PythonMonkey is a Mozilla [SpiderMonkey](https://firefox-source-docs.mozilla.org/js/index.html) JavaScript engine embedded into the Python VM,
 using the Python engine to provide the JS host environment.
 
-This product is in an early stage, approximately 75% to MVP as of May 2023. It is under active development by Distributive Corp.,
+This product is in an early stage, approximately 80% to MVP as of May 2023. It is under active development by Distributive Corp.,
 https://distributive.network/. External contributions and feedback are welcome and encouraged.
 
 The goal is to make writing code in either JS or Python a developer preference, with libraries commonly used in either language
@@ -33,12 +33,12 @@ this package to execute our complex `dcp-client` library, which is written in JS
 - [done] NodeJS+NPM-compatible CommonJS module system
 - [done] Python strings coerce to JS strings
 - [done] Python intrinsics coerce to JS intrinsics
-- Python dicts coerce to JS objects
+- [done] Python dicts coerce to JS objects
 - [done] Python `require` function, returns a coerced dict of module exports
 - [done] Python functions coerce to JS function wrappers
 - [done] CommonJS module system .py loader, loads Python modules for use by JS
 - JS object->Python dict coercion supports inherited-property lookup (via __getattribute__?)
-- Python host environment supplies event loop, including EventEmitter, setTimeout, etc.
+- [done] Python host environment supplies event loop, including EventEmitter, setTimeout, etc.
 - Python host environment supplies XMLHttpRequest (other project?)
 - Python host environment supplies basic subsets of NodeJS's fs, path, process, etc, modules; as-needed by dcp-client (other project?)
 - Python TypedArrays coerce to JS TypeArrays
@@ -55,20 +55,17 @@ this package to execute our complex `dcp-client` library, which is written in JS
     - spidermonkey 102.2.0 or later
     - npm (nodejs)
     - [Poetry](https://python-poetry.org/docs/#installation)
+    - [poetry-dynamic-versioning](https://github.com/mtkennerly/poetry-dynamic-versioning)
 
-2. Run `poetry install`. This command automatically compiles the project and installs the project as well as all Python dependencies to the poetry virtualenv.
+2. Run `poetry run pip install --verbose python/pminit ./`. This command automatically compiles the project and installs the project as well as dependencies into the poetry virtualenv.
 
 ## Running tests
 1. Compile the project 
-2. From the root directory, run `poetry run pytest ./tests/python`
+2. Install development dependencies: `poetry install --no-root --only=dev`
+3. From the root directory, run `poetry run pytest ./tests/python`
 
 ## Using the library
 
-See also: examples/
-
-### Method 1
-After compiling the project in the `build/src` folder you will find a `.so` file named `pythonmonkey.so`. This is the shared object file that contains the pythonmonkey module.
-
 ### Install from [PyPI](https://pypi.org/project/pythonmonkey/)
 
 > PythonMonkey is not release-ready yet. Our first public release is scheduled for mid-June 2023.
@@ -101,10 +98,19 @@ Type "help", "copyright", "credits" or "license" for more information.
 
 Alternatively, you can build a `wheel` package by running `poetry build --format=wheel`, and install it by `pip install dist/*.whl`.
 
+## Examples
+
+* [examples/](examples/)
+* https://github.com/Distributive-Network/PythonMonkey-examples
+* https://github.com/Distributive-Network/PythonMonkey-Crypto-JS-Fullstack-Example
+
 # Troubleshooting Tips
 
+## REPL - pmjs 
+A basic JavaScript shell, `pmjs`, ships with PythonMonkey.
+
 ## CommonJS (require)
-If you are having trouble with the CommonJS require function, set DEBUG='ctx-module*' and you can see the filenames it tries to laod
+If you are having trouble with the CommonJS require function, set environment variable DEBUG='ctx-module*' and you can see the filenames it tries to laod.
 
 ### Extra Symbols
 Loading the CommonJS subsystem declares some extra symbols which may be helpful in debugging -

build_script.sh

@@ -23,7 +23,3 @@ else
   cmake .. 
 fi
 cmake --build . -j$CPUS --config Release
-
-# npm is used to load JS components, see package.json
-cd "${topDir}/python/pythonmonkey/"
-npm i

include/JSObjectProxy.hh

@@ -57,15 +57,6 @@ public:
    */
   static int JSObjectProxy_init(JSObjectProxy *self, PyObject *args, PyObject *kwds);
 
-  /**
-   * @brief Helper function for JSObjectProxy_init
-   *
-   * @param jsObject - The underlying backing store JSObject for the JSObjectProxy
-   * @param dict - The python dict to be converted into a JSObjectProxy
-   * @param subValsMap - A map of PyObject to JS::Value pairs, representing values that have been visited so far
-   */
-  static void JSObjectProxy_init_helper(JS::HandleObject jsObject, PyObject *dict, std::unordered_map<PyObject *, JS::RootedValue *> &subValsMap);
-
   /**
    * @brief Length method (.mp_length), returns the number of key-value pairs in the JSObject, used by the python len() method
    *
@@ -121,16 +112,6 @@ public:
    * @return bool - Whether the compared objects are equal or not
    */
   static bool JSObjectProxy_richcompare_helper(JSObjectProxy *self, PyObject *other, std::unordered_map<PyObject *, PyObject *> &visited);
-
-  /**
-   * @brief Function to satisfy the python GC. See comment in function implementation for more details
-   *
-   * @param self - unused
-   * @param visit - unused
-   * @param arg - unused
-   * @return int - always 0
-   */
-  static int JSObjectProxy_traverse(JSObjectProxy *self, visitproc visit, void *arg);
 };
 
 
@@ -146,22 +127,5 @@ static PyMappingMethods JSObjectProxy_mapping_methods = {
 
 /**
  * @brief Struct for the JSObjectProxyType, used by all JSObjectProxy objects
- *
  */
-static PyTypeObject JSObjectProxyType = {
-  .ob_base = PyVarObject_HEAD_INIT(&PyType_Type, 0)
-  .tp_name = "pythonmonkey.JSObjectProxy",
-  .tp_basicsize = sizeof(JSObjectProxy),
-  .tp_dealloc = (destructor)JSObjectProxyMethodDefinitions::JSObjectProxy_dealloc,
-  .tp_as_mapping = &JSObjectProxy_mapping_methods,
-  .tp_getattro = (getattrofunc)JSObjectProxyMethodDefinitions::JSObjectProxy_get,
-  .tp_setattro = (setattrofunc)JSObjectProxyMethodDefinitions::JSObjectProxy_assign,
-  .tp_flags = Py_TPFLAGS_DEFAULT
-  | Py_TPFLAGS_DICT_SUBCLASS,  // https://docs.python.org/3/c-api/typeobj.html#Py_TPFLAGS_DICT_SUBCLASS
-  .tp_doc = PyDoc_STR("Javascript Object proxy dict"),
-  .tp_traverse = (traverseproc)JSObjectProxyMethodDefinitions::JSObjectProxy_traverse,
-  .tp_richcompare = (richcmpfunc)JSObjectProxyMethodDefinitions::JSObjectProxy_richcompare,
-  .tp_base = &PyDict_Type,
-  .tp_init = (initproc)JSObjectProxyMethodDefinitions::JSObjectProxy_init,
-  .tp_new = JSObjectProxyMethodDefinitions::JSObjectProxy_new,
-};
+extern PyTypeObject JSObjectProxyType;