Write our own minimum copy of npm in Python to remove dependency on Node.js during installation #1804
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'Test and Publish Suite' | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - '*' | |
| workflow_call: | |
| workflow_dispatch: | |
| inputs: | |
| debug_enabled_os: | |
| type: choice | |
| description: Optionally, choose an OS to run the build with SSH debugging on (https://github.com/fawazahmed0/action-debug) | |
| required: false | |
| options: | |
| - '' | |
| - 'ubuntu-20.04' | |
| - 'macos-12' | |
| - 'macos-14' | |
| - 'windows-2022' | |
| debug_enabled_python: | |
| type: choice | |
| description: Choose a Python version to run the build with SSH debugging on | |
| required: false | |
| options: | |
| - '' | |
| - '3.8' | |
| - '3.9' | |
| - '3.10' | |
| - '3.11' | |
| - '3.12' | |
| build_type: | |
| type: choice | |
| description: 'Choose the build type to use' | |
| required: false | |
| default: 'Debug' | |
| options: | |
| - 'Debug' | |
| - 'Profile' | |
| - 'Sanitize' | |
| - 'DRelease' | |
| - 'Release' | |
| - 'None' | |
| pull_request: | |
| env: | |
| # don't upgrade outdated brew packages because the process is too slow | |
| HOMEBREW_NO_INSTALL_UPGRADE: 1 | |
| HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 | |
| defaults: | |
| run: | |
| # run with Git Bash on Windows | |
| shell: bash | |
| jobs: | |
| build-spidermonkey-unix: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Use Ubuntu 20.04 / macOS 13 x86_64 / macOS 14 arm64 + Python 3.10 to build SpiderMonkey | |
| os: [ 'ubuntu-20.04', 'macos-13', 'macos-14', 'pi' ] # macOS 14 runner exclusively runs on M1 hardwares | |
| # see https://github.blog/changelog/2024-01-30-github-actions-macos-14-sonoma-is-now-available | |
| python_version: [ '3.10' ] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python_version }} | |
| - name: Read the mozilla-central commit hash to be used | |
| run: echo "MOZCENTRAL_VERSION=$(cat mozcentral.version)" >> $GITHUB_ENV | |
| - name: Cache spidermonkey build | |
| id: cache-spidermonkey | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ./_spidermonkey_install/* | |
| key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }} | |
| lookup-only: true # skip download | |
| - name: Setup XCode | |
| if: ${{ (matrix.os == 'macos-13' || matrix.os == 'macos-14') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} | |
| # SpiderMonkey requires XCode SDK version at least 13.3 | |
| run: sudo xcode-select -switch /Applications/Xcode_14.3.app | |
| - name: Build spidermonkey | |
| if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }} | |
| run: ./setup.sh | |
| build-spidermonkey-win: | |
| runs-on: windows-2022 | |
| # SpiderMonkey requires Visual Studio 2022 or newer. | |
| # The Windows 2019 runner only has Visual Studio Enterprise 2019 installed. | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Read the mozilla-central commit hash to be used | |
| run: echo "MOZCENTRAL_VERSION=$(cat mozcentral.version)" >> $GITHUB_ENV | |
| - name: Cache spidermonkey build | |
| id: cache-spidermonkey | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ./_spidermonkey_install/* | |
| key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }} | |
| lookup-only: true # skip download | |
| - name: Install dependencies | |
| if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }} | |
| shell: powershell | |
| run: | | |
| # Already installed in Github Actions runner | |
| # choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' # add CMake to system PATH | |
| # choco install -y llvm gnuwin32-m4 | |
| choco install -y wget make unzip | |
| - name: Install MozillaBuild | |
| if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }} | |
| run: | | |
| wget -c -q https://ftp.mozilla.org/pub/mozilla/libraries/win32/MozillaBuildSetup-Latest.exe | |
| powershell -command 'Start-Process -Wait -FilePath "./MozillaBuildSetup-Latest.exe" -ArgumentList "/S"' | |
| - name: Build spidermonkey in MozillaBuild environment | |
| if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }} | |
| env: | |
| # Preserve MozillaBuild v4.0.x behaviour | |
| # see https://groups.google.com/u/1/a/mozilla.org/g/dev-platform/c/hF51Q3j6ca8 | |
| USE_MINTTY: 0 | |
| run: /c/mozilla-build/start-shell.bat -use-full-path -here ./setup.sh | |
| build-and-test: | |
| needs: [build-spidermonkey-unix, build-spidermonkey-win] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ 'ubuntu-20.04', 'macos-12', 'macos-14', 'windows-2022', 'pi' ] | |
| python_version: [ '3.8', '3.9', '3.10', '3.11', '3.12' ] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 # fetch all history for all branches and tags | |
| # poetry-dynamic-versioning needs git tags to produce the correct version number | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python_version }} | |
| - name: Remove old poetry cache | |
| run: rm -rf ~/.cache/pypoetry | |
| if: ${{ matrix.os == 'pi' }} | |
| - name: Setup Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: 1.5.1 | |
| - name: Install Dependencies | |
| run: | | |
| echo "Installing Dependencies" | |
| if [[ "$OSTYPE" == "linux-gnu"* ]]; then # Linux | |
| sudo apt-get update -y | |
| sudo apt-get install -y cmake llvm | |
| elif [[ "$OSTYPE" == "darwin"* ]]; then # macOS | |
| brew update || true # allow failure | |
| brew install cmake pkg-config wget unzip coreutils # `coreutils` installs the `realpath` command | |
| fi | |
| echo "Installing python deps" | |
| poetry self add "poetry-dynamic-versioning[plugin]" | |
| poetry env use python$PYTHON_VERSION || poetry env use python3 # use the correct Python version we've set up | |
| poetry install --no-root --only=dev | |
| echo "Installed Dependencies" | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python_version }} | |
| - name: Build Docs # only build docs once | |
| if: ${{ matrix.os == 'ubuntu-20.04' && matrix.python_version == '3.11' }} | |
| run: | | |
| sudo apt-get install -y graphviz | |
| # the newest version in Ubuntu 20.04 repository is 1.8.17, but we need Doxygen 1.9 series | |
| wget -c -q https://www.doxygen.nl/files/doxygen-1.9.7.linux.bin.tar.gz | |
| tar xf doxygen-1.9.7.linux.bin.tar.gz | |
| cd doxygen-1.9.7 && sudo make install && cd - | |
| rm -rf doxygen-1.9.7 doxygen-1.9.7.linux.bin.tar.gz | |
| BUILD_DOCS=1 BUILD_TYPE=None poetry install | |
| - name: Upload Doxygen-generated docs as CI artifacts | |
| if: ${{ matrix.os == 'ubuntu-20.04' && matrix.python_version == '3.11' }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: docs-${{ github.run_id }}-${{ github.sha }} | |
| path: ./build/docs/html/ | |
| - name: Read the mozilla-central commit hash to be used | |
| run: echo "MOZCENTRAL_VERSION=$(cat mozcentral.version)" >> $GITHUB_ENV | |
| - name: Use cached spidermonkey build | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ./_spidermonkey_install/* | |
| key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }} | |
| fail-on-cache-miss: true # SpiderMonkey is expected to be cached in its dedicated job | |
| - name: Build pminit | |
| run: | | |
| cd python/pminit | |
| poetry build --format=sdist | |
| cd - | |
| mkdir -p ./dist/ | |
| mv -v python/pminit/dist/* ./dist/ | |
| - name: Build wheel | |
| run: | | |
| echo $(poetry run python --version) | |
| WORKFLOW_BUILD_TYPE=${{ inputs.build_type }} | |
| BUILD_TYPE=${WORKFLOW_BUILD_TYPE:-"Debug"} poetry build --format=wheel | |
| ls -lah ./dist/ | |
| - name: Make the wheels we build also support lower versions of macOS | |
| if: ${{ matrix.os == 'macos-12' || matrix.os == 'macos-14' }} | |
| # Change the platform tag part of the wheel filename to `macosx_11_0_xxx` (means to support macOS 11.0 and above) | |
| # See https://packaging.python.org/en/latest/specifications/binary-distribution-format/#file-format | |
| # A wheel package file will only be selected by pip to install if the platform tag satisfies, regardless of whether the binary compatibility actually is. | |
| # Otherwise, pip would fallback to compile from the source distribution. | |
| run: | | |
| cd ./dist/ | |
| for file in *.whl; do | |
| mv "$file" "$(echo "$file" | sed -E 's/macosx_[0-9]+_[0-9]+/macosx_11_0/')"; | |
| done | |
| - name: Upload wheel as CI artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheel-${{ github.run_id }}-${{ github.sha }} | |
| path: ./dist/ | |
| - name: Set cores to get stored in /cores | |
| if: ${{ matrix.os != 'windows-2022' }} | |
| # TODO (Caleb Aikens) figure out how to get Windows core dumps | |
| run: | | |
| sudo mkdir -p /cores | |
| sudo chmod 777 /cores | |
| # Core filenames will be of the form osname.pythonversion.executable.pid.timestamp: | |
| if [[ "$OSTYPE" == "linux-gnu"* ]]; then | |
| sudo bash -c 'echo "/cores/${OSTYPE}.$(poetry run python --version).%e.%p.%t" > /proc/sys/kernel/core_pattern' | |
| else | |
| sudo sysctl kern.corefile="/cores/${OSTYPE}.$(poetry run python --version).%e.%p.%y" | |
| fi | |
| - name: Run Python tests (pytest) | |
| run: | | |
| if [[ "$OSTYPE" == "linux-gnu"* || "$OSTYPE" == "darwin"* ]]; then | |
| # TODO (Caleb Aikens) figure out how to get Windows core dumps | |
| ulimit -c unlimited | |
| fi | |
| WORKFLOW_BUILD_TYPE=${{ inputs.build_type }} | |
| BUILD_TYPE=${WORKFLOW_BUILD_TYPE:-"Debug"} poetry run python -m pip install --force-reinstall --verbose ./dist/* | |
| poetry run python -m pytest tests/python | |
| - name: Run JS tests (peter-jr) | |
| if: ${{ (success() || failure()) }} | |
| run: | | |
| if [[ "$OSTYPE" == "linux-gnu"* || "$OSTYPE" == "darwin"* ]]; then | |
| # TODO (Caleb Aikens) figure out how to get Windows core dumps | |
| ulimit -c unlimited | |
| fi | |
| poetry run bash ./peter-jr ./tests/js/ | |
| - name: SSH debug session | |
| if: ${{ (success() || failure()) && github.event_name == 'workflow_dispatch' && inputs.debug_enabled_os == matrix.os && inputs.debug_enabled_python == matrix.python_version}} | |
| uses: fawazahmed0/action-debug@main | |
| with: | |
| credentials: "admin:admin" | |
| - name: Upload core dumps as CI artifacts | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ matrix.os != 'windows-2022' && failure() }} | |
| # TODO (Caleb Aikens) figure out how to get Windows core dumps | |
| with: | |
| name: cores-${{ matrix.os }}-${{ matrix.python_version }} | |
| path: /cores | |
| sdist: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - name: Setup Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: 1.5.1 | |
| - name: Build source distribution (sdist) file | |
| run: | | |
| poetry self add "poetry-dynamic-versioning[plugin]" | |
| poetry build --format=sdist | |
| ls -lah ./dist/ | |
| - name: Upload sdist as CI artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheel-${{ github.run_id }}-${{ github.sha }} | |
| path: ./dist/ | |
| check-install-from-sdist: | |
| needs: sdist | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Download wheels built | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: wheel-${{ github.run_id }}-${{ github.sha }} | |
| path: ./dist/ | |
| - run: pip install ./dist/pythonmonkey-*.tar.gz | |
| publish: | |
| needs: [build-and-test, sdist] | |
| runs-on: ubuntu-20.04 | |
| if: ${{ success() && github.event_name == 'push' && contains(github.ref, 'refs/tags/') }} | |
| steps: | |
| # no need to checkout | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - run: pip install twine | |
| - name: Download wheels built | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: wheel-${{ github.run_id }}-${{ github.sha }} | |
| path: ./dist/ | |
| - run: ls -lah ./dist/ | |
| - name: Publish package | |
| run: | | |
| twine upload dist/* \ | |
| --non-interactive --skip-existing \ | |
| --username __token__ --password ${{ secrets.PYPI_API_TOKEN }} | |
| publish-nightly: | |
| # Implement a very basic Python package repository (https://peps.python.org/pep-0503/) | |
| # and deploy the static files to GitHub Pages | |
| needs: [build-and-test, sdist] | |
| runs-on: ubuntu-20.04 | |
| if: ${{ (success() || failure()) && github.ref_name == 'main' }} # publish nightly builds regardless of tests failure | |
| permissions: # grant GITHUB_TOKEN the permissions required to make a Pages deployment | |
| pages: write | |
| id-token: write | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deployment.outputs.page_url }} | |
| steps: | |
| # don't checkout | |
| - name: Download wheels built | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: wheel-${{ github.run_id }}-${{ github.sha }} | |
| path: ./dist/ | |
| - name: Download docs html generated by Doxygen | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: docs-${{ github.run_id }}-${{ github.sha }} | |
| path: ./docs/ | |
| - name: Move wheels to the correct repository project paths | |
| run: | | |
| mkdir -p ./pythonmonkey/ ./pminit/ | |
| mv ./dist/pythonmonkey-* ./pythonmonkey/ | |
| mv ./dist/pminit-* ./pminit/ | |
| - name: Generate index page for the project | |
| run: | | |
| generate_index () { | |
| cd $1 | |
| html="<html><head><title>PythonMonkey Nightly Builds</title></head><body>" | |
| for file in ./*; do # generate <a> tags for each file | |
| html+="<a href=\"$file\">$file</a><br/>" | |
| done | |
| html+="</body></html>" | |
| echo "$html" > ./index.html | |
| cd - | |
| } | |
| generate_index ./pythonmonkey/ | |
| generate_index ./pminit/ | |
| - name: Generate repository root page | |
| run: | | |
| html="<html><head><title>PythonMonkey Nightly Builds</title></head><body>" | |
| html+="<h1>PythonMonkey Nightly Builds</h1>" | |
| html+="<p>To install nightly builds, run</p>" | |
| html+="<pre>pip install --extra-index-url https://nightly.pythonmonkey.io/ --pre pythonmonkey</pre>" | |
| html+="<h3>Browse files:</h3>" | |
| html+="<li><a href="pythonmonkey/">pythonmonkey</a></li>" | |
| html+="<li><a href="pminit/">pminit</a></li>" | |
| html+="</body></html>" | |
| echo "$html" > ./index.html | |
| - uses: actions/upload-pages-artifact@v1 | |
| with: | |
| path: ./ | |
| - name: Deploy to GitHub Pages | |
| id: deployment | |
| uses: actions/deploy-pages@v2 |