Merge pull request #367 from Distributive-Network/Xmader/chore/moz-central

Build SpiderMonkey from `mozilla-central`

Author: philippedistributive

.github/workflows/test-and-publish.yaml

@@ -18,7 +18,7 @@ on:
         - 'ubuntu-20.04'
         - 'macos-12'
         - 'macos-14'
-        - 'windows-2019'
+        - 'windows-2022'
       debug_enabled_python:
         type: choice
         description: Choose a Python version to run the build with SSH debugging on
@@ -69,13 +69,15 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python_version }}
+      - name: Read the mozilla-central commit hash to be used
+        run: echo "MOZCENTRAL_VERSION=$(cat mozcentral.version)" >> $GITHUB_ENV
       - name: Cache spidermonkey build
         id: cache-spidermonkey
         uses: actions/cache@v4
         with:
           path: |
             ./_spidermonkey_install/*
-          key: spidermonkey115.8.0-${{ runner.os }}-${{ runner.arch }}
+          key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }}
           lookup-only: true # skip download
       - name: Setup XCode
         if: ${{ (matrix.os == 'macos-13' || matrix.os == 'macos-14') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
@@ -86,16 +88,20 @@ jobs:
         if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
         run: ./setup.sh
   build-spidermonkey-win:
-    runs-on: windows-2019
+    runs-on: windows-2022
+      # SpiderMonkey requires Visual Studio 2022 or newer.
+      # The Windows 2019 runner only has Visual Studio Enterprise 2019 installed.
     steps:
       - uses: actions/checkout@v4
+      - name: Read the mozilla-central commit hash to be used
+        run: echo "MOZCENTRAL_VERSION=$(cat mozcentral.version)" >> $GITHUB_ENV
       - name: Cache spidermonkey build
         id: cache-spidermonkey
         uses: actions/cache@v4
         with:
           path: |
             ./_spidermonkey_install/*
-          key: spidermonkey115.8.0-${{ runner.os }}-${{ runner.arch }}
+          key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }}
           lookup-only: true # skip download
       - name: Install dependencies
         if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
@@ -104,7 +110,7 @@ jobs:
           # Already installed in Github Actions runner
           # choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' # add CMake to system PATH
           # choco install -y llvm gnuwin32-m4
-          choco install -y wget make
+          choco install -y wget make unzip
       - name: Install MozillaBuild
         if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
         run: |
@@ -123,7 +129,7 @@ jobs:
       fail-fast: false
       matrix:
         # The lowest supported version is Ubuntu 20.04 + Python 3.8 or macOS 12 + Python 3.9
-        os: [ 'ubuntu-20.04', 'macos-12', 'macos-14', 'windows-2019' ]
+        os: [ 'ubuntu-20.04', 'macos-12', 'macos-14', 'windows-2022' ]
         python_version: [ '3.8', '3.9', '3.10', '3.11', '3.12' ]
         exclude:
           # actions/setup-python: The version '3.8'/'3.9' with architecture 'arm64' was not found for macOS.
@@ -154,7 +160,7 @@ jobs:
             sudo apt-get install -y cmake llvm
           elif [[ "$OSTYPE" == "darwin"* ]]; then # macOS
             brew update || true # allow failure
-            brew install cmake pkg-config wget coreutils # `coreutils` installs the `realpath` command
+            brew install cmake pkg-config wget unzip coreutils # `coreutils` installs the `realpath` command
           fi
           echo "Installing python deps"
           poetry self add "poetry-dynamic-versioning[plugin]"
@@ -177,12 +183,14 @@ jobs:
         with:
           name: docs-${{ github.run_id }}-${{ github.sha }}
           path: ./build/docs/html/
+      - name: Read the mozilla-central commit hash to be used
+        run: echo "MOZCENTRAL_VERSION=$(cat mozcentral.version)" >> $GITHUB_ENV
       - name: Use cached spidermonkey build
         uses: actions/cache@v4
         with:
           path: |
             ./_spidermonkey_install/*
-          key: spidermonkey115.8.0-${{ runner.os }}-${{ runner.arch }}
+          key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }}
           fail-on-cache-miss: true # SpiderMonkey is expected to be cached in its dedicated job
       - name: Build pminit
         run: |
@@ -203,7 +211,7 @@ jobs:
           name: wheel-${{ github.run_id }}-${{ github.sha }}
           path: ./dist/
       - name: Set cores to get stored in /cores
-        if: ${{ matrix.os != 'windows-2019' }}
+        if: ${{ matrix.os != 'windows-2022' }}
         # TODO (Caleb Aikens) figure out how to get Windows core dumps
         run: |
           sudo mkdir -p /cores
@@ -238,7 +246,7 @@ jobs:
           credentials: "admin:admin"
       - name: Upload core dumps as CI artifacts
         uses: actions/upload-artifact@v3
-        if: ${{ matrix.os != 'windows-2019' && failure() }}
+        if: ${{ matrix.os != 'windows-2022' && failure() }}
         # TODO (Caleb Aikens) figure out how to get Windows core dumps
         with:
           name: cores-${{ matrix.os }}-${{ matrix.python_version }}

.github/workflows/update-mozcentral-version.yaml

@@ -0,0 +1,39 @@
+name: 'Create pull requests to update mozilla-central version to the latest'
+
+on:
+  schedule:
+    - cron: "00 14 * * 1" # run every Monday at 14:00 UTC (10:00 Eastern Daylight Time)
+  workflow_call:
+  workflow_dispatch: # or you can run it manually
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  update:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check Version
+        # Check the latest changes on mozilla-central via the Mercurial pushlog HTTP API
+        # See https://mozilla-version-control-tools.readthedocs.io/en/latest/hgmo/pushlog.html#hgweb-commands
+        run: |
+          COMMIT_HASH=$(
+            curl -s "https://hg.mozilla.org/mozilla-central/json-pushes?tipsonly=1&version=2" |\
+            jq --join-output '(.lastpushid | tostring) as $pushid | empty, .pushes[$pushid].changesets[0]'
+          )
+          echo "MOZCENTRAL_VERSION=$COMMIT_HASH" >> $GITHUB_ENV
+      - name: Update `mozcentral.version` File
+        run: echo $MOZCENTRAL_VERSION > mozcentral.version
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          add-paths: mozcentral.version
+          commit-message: |
+            chore(deps): upgrade SpiderMonkey to `${{ env.MOZCENTRAL_VERSION }}`
+          branch: chore/upgrade-spidermonkey-to-${{ env.MOZCENTRAL_VERSION }}
+          title: Upgrade SpiderMonkey to mozilla-central commit `${{ env.MOZCENTRAL_VERSION }}`
+          body: |
+            Changeset: https://hg.mozilla.org/mozilla-central/rev/${{ env.MOZCENTRAL_VERSION }}
+          labels: dependencies

.gitignore

@@ -7,10 +7,12 @@ lib/*
 .pytest_cache
 .DS_Store
 firefox-*.tar.xz
+firefox-*.zip
 firefox-*/
+mozilla-central-*
 __pycache__
 Testing/Temporary
-_spidermonkey_install
+_spidermonkey_install*
 uncrustify-*.tar.gz
 uncrustify-*/
 uncrustify

cmake/modules/FindSpiderMonkey.cmake

@@ -24,8 +24,6 @@
 #  SPIDERMONKEY_FOUND		  - True if SpiderMonkey found.
 #  SPIDERMONKEY_THREADSAFE	 - True if SpiderMonkey is compiled with multi threading support.
 
-#Last Change: 2022-10-03 (Caleb Aikens)
-
 include(CheckIncludeFileCXX)
 include(CheckCXXSourceCompiles)
 include(CheckCXXSourceRuns)
@@ -36,11 +34,26 @@ if(SPIDERMONKEY_FOUND)
 	set(SPIDERMONKEY_FIND_QUIETLY TRUE)
 endif()
 
+# Get the SpiderMonkey major version number
+# See https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/src/old-configure.in#l1081
+file(GLOB LIB_PATH "${CMAKE_CURRENT_SOURCE_DIR}/_spidermonkey_install")
+execute_process(COMMAND 
+	"sh" "-c" "./js*-config --version" # Run "_spidermonkey_install/bin/js*-config --version" to print the full version number 
+	WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/_spidermonkey_install/bin"
+ 	OUTPUT_VARIABLE MOZILLA_VERSION
+)
+string(STRIP ${MOZILLA_VERSION} MOZILLA_VERSION)
+string(REGEX REPLACE "^([0-9]+)(\\.[0-9]+)*([ab][0-9]|)?" # Only the MAJOR and the "a1" (indicator of nightly build) part is needed
+                     "\\1\\3"                             # see https://hg.mozilla.org/releases/mozilla-esr102/file/tip/build/moz.configure/init.configure#l959
+	MOZILLA_SYMBOLVERSION
+	${MOZILLA_VERSION}
+)
+
 # SpiderMonkey search paths
 set(SPIDERMONKEY_PATHS
   "${CMAKE_CURRENT_SOURCE_DIR}/_spidermonkey_install"
   "${CMAKE_CURRENT_SOURCE_DIR}/_spidermonkey_install/lib"
-  "${CMAKE_CURRENT_SOURCE_DIR}/_spidermonkey_install/include/mozjs-115"
+  "${CMAKE_CURRENT_SOURCE_DIR}/_spidermonkey_install/include/mozjs-${MOZILLA_SYMBOLVERSION}"
   ${SPIDERMONKEY_ROOT}
 	$ENV{SPIDERMONKEY_ROOT}
 	~/Library/Frameworks
@@ -60,7 +73,7 @@ set(SPIDERMONKEY_PATHS
 set(SPIDERMONKEY_HEADERS jsapi.h js/RequiredDefines.h)
 
 # SpiderMonkey include suffix paths
-set(SPIDERMONKEY_INCLUDE_SUFFIX_PATHS include/mozjs-115/)
+set(SPIDERMONKEY_INCLUDE_SUFFIX_PATHS include/mozjs-${MOZILLA_SYMBOLVERSION}/)
 
 # Find SpiderMonkey include path
 find_path(SPIDERMONKEY_INCLUDE_DIR ${SPIDERMONKEY_HEADERS}
@@ -71,7 +84,7 @@ find_path(SPIDERMONKEY_INCLUDE_DIR ${SPIDERMONKEY_HEADERS}
 )
 
 # SpiderMonkey libs
-set(SPIDERMONKEY_LIBRARY_NAMES libmozjs-115.so libmozjs-115.dylib mozjs-115.lib)
+set(SPIDERMONKEY_LIBRARY_NAMES libmozjs-${MOZILLA_SYMBOLVERSION}.so libmozjs-${MOZILLA_SYMBOLVERSION}.dylib mozjs-${MOZILLA_SYMBOLVERSION}.lib)
 
 set(SPIDERMONKEY_LIB_SUFFIX_PATHS js/src/build lib)
 

include/JobQueue.hh

@@ -78,6 +78,11 @@ void runJobs(JSContext *cx) override;
  */
 bool empty() const override;
 
+/**
+ * @return true if the job queue stopped draining, which results in `empty()` being false after `runJobs()`.
+ */
+bool isDrainingStopped() const override;
+
 /**
  * @brief Appends a callback to the queue of FinalizationRegistry callbacks
  *

include/jsTypeFactory.hh

@@ -25,15 +25,18 @@ public:
    * @return PyObject* - the PyObject string
    */
   static PyObject *getPyString(const char16_t *chars);
+  static PyObject *getPyString(const JS::Latin1Char *chars);
 
   /**
    * @brief decrefs the underlying PyObject string when the JSString is finalized
    *
    * @param chars - The char buffer of the string
    */
   void finalize(char16_t *chars) const override;
+  void finalize(JS::Latin1Char *chars) const override;
 
   size_t sizeOfBuffer(const char16_t *chars, mozilla::MallocSizeOf mallocSizeOf) const override;
+  size_t sizeOfBuffer(const JS::Latin1Char *chars, mozilla::MallocSizeOf mallocSizeOf) const override;
 };
 extern PythonExternalString PythonExternalStringCallbacks;
 

mozcentral.version

@@ -0,0 +1 @@
+d25c9bdacb64ae50779dc91f5919ca0189ad6c36

pyproject.toml

@@ -23,6 +23,7 @@ include = [
   { path = "tests", format = "sdist" },
   { path = "CMakeLists.txt", format = "sdist" },
   { path = "*.sh", format = "sdist" },
+  { path = "mozcentral.version", format = "sdist" },
 ]
 
 

python/pythonmonkey/lib/pmdb.py

@@ -22,7 +22,7 @@ def enable(debuggerGlobalObject=pm.eval("debuggerGlobal")):
     return  # already enabled, skipping
 
   debuggerGlobalObject._pmdbEnabled = True
-  debuggerGlobalObject.eval("""(debuggerInput, _pythonPrint, _pythonExit) => {
+  pm.eval("debuggerGlobal.eval")("""(mainGlobal, debuggerInput, _pythonPrint, _pythonExit) => {
   const dbg = new Debugger()
   const mainDebuggee = dbg.addDebuggee(mainGlobal)
   dbg.uncaughtExceptionHook = (e) => {
@@ -177,4 +177,4 @@ def enable(debuggerGlobalObject=pm.eval("debuggerGlobal")):
   // Enter debugger on `debugger;` statement
   dbg.onDebuggerStatement = (frame) => enterDebuggerLoop(frame)
 
-  }""")(debuggerInput, print, lambda status: exit(int(status)))
+  }""")(pm.globalThis, debuggerInput, print, lambda status: exit(int(status)))

setup.sh

@@ -18,7 +18,7 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then # Linux
     SUDO='sudo'
   fi
   $SUDO apt-get update --yes
-  $SUDO apt-get install --yes cmake graphviz llvm clang pkg-config m4 \
+  $SUDO apt-get install --yes cmake graphviz llvm clang pkg-config m4 unzip \
     wget curl python3-distutils python3-dev
   # Install Doxygen
   # the newest version in Ubuntu 20.04 repository is 1.8.17, but we need Doxygen 1.9 series
@@ -28,15 +28,16 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then # Linux
   rm -rf doxygen-1.9.7 doxygen-1.9.7.linux.bin.tar.gz
 elif [[ "$OSTYPE" == "darwin"* ]]; then # macOS
   brew update || true # allow failure
-  brew install cmake doxygen pkg-config wget coreutils # `coreutils` installs the `realpath` command
+  brew install cmake doxygen pkg-config wget unzip coreutils # `coreutils` installs the `realpath` command
 elif [[ "$OSTYPE" == "msys"* ]]; then # Windows
   echo "Dependencies are not going to be installed automatically on Windows."
 else
   echo "Unsupported OS"
   exit 1
 fi
 # Install rust compiler
-curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.69 # force to use Rust 1.69 because 1.70 has linking issues on Windows
+curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.76
+cargo install cbindgen
 # Setup Poetry
 curl -sSL https://install.python-poetry.org | python3 - --version "1.7.1"
 if [[ "$OSTYPE" == "msys"* ]]; then # Windows
@@ -51,15 +52,15 @@ echo "Done installing dependencies"
 echo "Downloading uncrustify source code"
 wget -c -q https://github.com/uncrustify/uncrustify/archive/refs/tags/uncrustify-0.78.1.tar.gz
 mkdir -p uncrustify-source
-tar -xzvf uncrustify-0.78.1.tar.gz -C uncrustify-source --strip-components=1 # strip the root folder
+tar -xzf uncrustify-0.78.1.tar.gz -C uncrustify-source --strip-components=1 # strip the root folder
 echo "Done downloading uncrustify source code"
 
 echo "Building uncrustify"
 cd uncrustify-source
 mkdir -p build
 cd build
 if [[ "$OSTYPE" == "msys"* ]]; then # Windows
-  cmake ../ -T ClangCL
+  cmake ../
   cmake --build . -j$CPUS --config Release
   cp Release/uncrustify.exe ../../uncrustify.exe
 else
@@ -71,9 +72,10 @@ cd ../..
 echo "Done building uncrustify"
 
 echo "Downloading spidermonkey source code"
-wget -c -q https://ftp.mozilla.org/pub/firefox/releases/115.8.0esr/source/firefox-115.8.0esr.source.tar.xz
-mkdir -p firefox-source
-tar xf firefox-115.8.0esr.source.tar.xz -C firefox-source --strip-components=1 # strip the root folder
+# Read the commit hash for mozilla-central from the `mozcentral.version` file
+MOZCENTRAL_VERSION=$(cat mozcentral.version)
+wget -c -q -O firefox-source-${MOZCENTRAL_VERSION}.zip https://hg.mozilla.org/mozilla-central/archive/${MOZCENTRAL_VERSION}.zip
+unzip -q firefox-source-${MOZCENTRAL_VERSION}.zip && mv mozilla-central-${MOZCENTRAL_VERSION} firefox-source
 echo "Done downloading spidermonkey source code"
 
 echo "Building spidermonkey"
@@ -85,14 +87,20 @@ sed -i'' -e '/"winheap.cpp"/d' ./memory/mozalloc/moz.build # https://bugzilla.mo
 sed -i'' -e 's/"install-name-tool"/"install_name_tool"/' ./moz.configure # `install-name-tool` does not exist, but we have `install_name_tool`
 sed -i'' -e 's/bool Unbox/JS_PUBLIC_API bool Unbox/g' ./js/public/Class.h           # need to manually add JS_PUBLIC_API to js::Unbox until it gets fixed in Spidermonkey
 sed -i'' -e 's/bool js::Unbox/JS_PUBLIC_API bool js::Unbox/g' ./js/src/vm/JSObject.cpp  # same here
+sed -i'' -e 's/shared_lib = self._pretty_path(libdef.output_path, backend_file)/shared_lib = libdef.lib_name/' ./python/mozbuild/mozbuild/backend/recursivemake.py
+sed -i'' -e 's/if version < Version(mac_sdk_min_version())/if False/' ./build/moz.configure/toolchain.configure # do not verify the macOS SDK version as the required version is not available on Github Actions runner
+sed -i'' -e 's/return JS::GetWeakRefsEnabled() == JS::WeakRefSpecifier::Disabled/return false/' ./js/src/vm/GlobalObject.cpp # forcibly enable FinalizationRegistry
+sed -i'' -e 's/return !IsIteratorHelpersEnabled()/return false/' ./js/src/vm/GlobalObject.cpp # forcibly enable iterator helpers
+sed -i'' -e '/MOZ_CRASH_UNSAFE_PRINTF/,/__PRETTY_FUNCTION__);/d' ./mfbt/LinkedList.h # would crash in Debug Build: in `~LinkedList()` it should have removed all this list's elements before the list's destruction
+sed -i'' -e '/MOZ_ASSERT(stackRootPtr == nullptr);/d' ./js/src/vm/JSContext.cpp # would assert false in Debug Build since we extensively use `new JS::Rooted`
 cd js/src
 mkdir -p _build
 cd _build
 mkdir -p ../../../../_spidermonkey_install/
 ../configure \
   --prefix=$(realpath $PWD/../../../../_spidermonkey_install) \
   --with-intl-api \
-  --without-system-zlib \
+  $(if [[ "$OSTYPE" != "msys"* ]]; then echo "--without-system-zlib"; fi) \
   --disable-debug-symbols \
   --disable-jemalloc \
   --disable-tests \