Improve LLVM Support #438
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python test / linting | |
| env: | |
| TRIGGER_ON_PR_PUSH: true # Set to true to enable triggers on PR pushes | |
| RUSTFLAGS: -C debuginfo=0 | |
| RUST_BACKTRACE: 1 | |
| PYTHONUTF8: 1 | |
| on: | |
| push: | |
| branches: [ "master", "development", "dev" ] | |
| pull_request: | |
| branches: [ "master", "development", "dev" ] | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| python-test: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macOS-latest] | |
| python-version: ["3.10", "3.11", "3.12", "3.13"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set up Visual Studio environment on Windows | |
| if: runner.os == 'Windows' | |
| uses: ilammy/msvc-dev-cmd@v1 | |
| with: | |
| arch: x64 | |
| - name: Install the latest version of uv | |
| uses: astral-sh/setup-uv@v4 | |
| with: | |
| enable-cache: true | |
| - name: Set up Rust | |
| run: rustup show | |
| - name: Cache Rust | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: python/pecos-rslib | |
| - name: Install LLVM 14.0.6 using pecos-llvm (Unix) | |
| if: runner.os != 'Windows' | |
| run: | | |
| echo "Installing LLVM using pecos-llvm-utils..." | |
| cargo run -p pecos-llvm-utils --bin pecos-llvm --release -- install | |
| echo "Setting LLVM environment variables..." | |
| export PECOS_LLVM=$(cargo run -p pecos-llvm-utils --bin pecos-llvm --release -- find 2>/dev/null) | |
| export LLVM_SYS_140_PREFIX="$PECOS_LLVM" | |
| echo "PECOS_LLVM=$PECOS_LLVM" >> $GITHUB_ENV | |
| echo "LLVM_SYS_140_PREFIX=$LLVM_SYS_140_PREFIX" >> $GITHUB_ENV | |
| echo "Verifying LLVM installation..." | |
| cargo run -p pecos-llvm-utils --bin pecos-llvm --release -- check | |
| - name: Install LLVM 14.0.6 using pecos-llvm (Windows) | |
| if: runner.os == 'Windows' | |
| shell: pwsh | |
| run: | | |
| Write-Host "Installing LLVM using pecos-llvm-utils..." | |
| cargo run -p pecos-llvm-utils --bin pecos-llvm --release -- install | |
| Write-Host "Setting LLVM environment variables..." | |
| $env:PECOS_LLVM = (cargo run -p pecos-llvm-utils --bin pecos-llvm --release -- find 2>$null) | |
| $env:LLVM_SYS_140_PREFIX = $env:PECOS_LLVM | |
| "PECOS_LLVM=$env:PECOS_LLVM" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | |
| "LLVM_SYS_140_PREFIX=$env:LLVM_SYS_140_PREFIX" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | |
| Write-Host "Verifying LLVM installation..." | |
| cargo run -p pecos-llvm-utils --bin pecos-llvm --release -- check | |
| - name: Build and test PECOS (Windows) | |
| if: runner.os == 'Windows' | |
| shell: pwsh | |
| run: | | |
| # Ensure LLVM environment variable is available | |
| Write-Host "LLVM_SYS_140_PREFIX: $env:LLVM_SYS_140_PREFIX" | |
| Write-Host "LLVM_PATH: $env:LLVM_PATH" | |
| # If LLVM_SYS_140_PREFIX is not set but LLVM_PATH is, use LLVM_PATH | |
| if (-not $env:LLVM_SYS_140_PREFIX -and $env:LLVM_PATH) { | |
| Write-Host "Setting LLVM_SYS_140_PREFIX from LLVM_PATH" | |
| $env:LLVM_SYS_140_PREFIX = $env:LLVM_PATH | |
| } | |
| # Double check it's really set | |
| if (-not $env:LLVM_SYS_140_PREFIX) { | |
| Write-Error "LLVM_SYS_140_PREFIX is still not set!" | |
| exit 1 | |
| } | |
| # Export to GitHub env for make/maturin to use (in case it wasn't already) | |
| "LLVM_SYS_140_PREFIX=$env:LLVM_SYS_140_PREFIX" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | |
| # Also ensure LLVM bin is in PATH | |
| $llvmBinDir = Join-Path -Path $env:LLVM_SYS_140_PREFIX -ChildPath "bin" | |
| if (Test-Path $llvmBinDir) { | |
| "$llvmBinDir" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append | |
| Write-Host "Added LLVM bin to PATH: $llvmBinDir" | |
| } | |
| # Verify LLVM installation | |
| Write-Host "Checking for llvm-config..." | |
| Get-Command llvm-config -ErrorAction SilentlyContinue || Write-Host "llvm-config not found in PATH" | |
| # List LLVM directory contents | |
| Write-Host "LLVM directory contents:" | |
| Get-ChildItem $env:LLVM_SYS_140_PREFIX -ErrorAction SilentlyContinue | Select-Object Name | |
| # Find MSVC link.exe and create cargo config | |
| $vsWhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" | |
| $vsPath = & $vsWhere -latest -property installationPath | |
| $linkPath = Get-ChildItem -Path "$vsPath\VC\Tools\MSVC" -Recurse -Filter "link.exe" | | |
| Where-Object { $_.FullName -like "*\bin\Hostx64\x64\*" } | | |
| Select-Object -First 1 -ExpandProperty FullName | |
| if ($linkPath) { | |
| Write-Host "Found MSVC link.exe at: $linkPath" | |
| # Create .cargo directory and config in multiple locations | |
| # Create in root | |
| New-Item -ItemType Directory -Force -Path .cargo | Out-Null | |
| # Create config with escaped path and LLVM environment variable | |
| $escapedPath = $linkPath.Replace('\', '/') | |
| $escapedLLVMPath = $env:LLVM_SYS_140_PREFIX.Replace('\', '/') | |
| $configContent = "[target.x86_64-pc-windows-msvc]`nlinker = `"$escapedPath`"`n`n[env]`nLLVM_SYS_140_PREFIX = `"$escapedLLVMPath`"" | |
| $configContent | Out-File -FilePath ".cargo\config.toml" -Encoding UTF8 | |
| # Also create in pecos-rslib directory | |
| New-Item -ItemType Directory -Force -Path "python\pecos-rslib\.cargo" | Out-Null | |
| $configContent | Out-File -FilePath "python\pecos-rslib\.cargo\config.toml" -Encoding UTF8 | |
| # And in the rust subdirectory | |
| New-Item -ItemType Directory -Force -Path "python\pecos-rslib\rust\.cargo" | Out-Null | |
| $configContent | Out-File -FilePath "python\pecos-rslib\rust\.cargo\config.toml" -Encoding UTF8 | |
| # Also create in user's cargo home directory as a fallback | |
| $cargoHome = if ($env:CARGO_HOME) { $env:CARGO_HOME } else { "$env:USERPROFILE\.cargo" } | |
| New-Item -ItemType Directory -Force -Path $cargoHome | Out-Null | |
| # For user cargo config, we need to be careful not to overwrite existing content | |
| if (Test-Path "$cargoHome\config.toml") { | |
| # Add our content to existing file | |
| "`n$configContent" | Out-File -FilePath "$cargoHome\config.toml" -Encoding UTF8 -Append | |
| } else { | |
| # Create new file | |
| $configContent | Out-File -FilePath "$cargoHome\config.toml" -Encoding UTF8 | |
| } | |
| Write-Host "Created cargo configs with LLVM_SYS_140_PREFIX=$escapedLLVMPath" | |
| Write-Host "Root .cargo\config.toml:" | |
| Get-Content .cargo\config.toml | |
| Write-Host "User cargo config appended to: $cargoHome\config.toml" | |
| } else { | |
| Write-Error "Could not find MSVC link.exe" | |
| exit 1 | |
| } | |
| # Ensure LLVM environment variable is exported for subprocess | |
| [System.Environment]::SetEnvironmentVariable("LLVM_SYS_140_PREFIX", $env:LLVM_SYS_140_PREFIX, "User") | |
| [System.Environment]::SetEnvironmentVariable("LLVM_SYS_140_PREFIX", $env:LLVM_SYS_140_PREFIX, "Process") | |
| # Also set it as a regular environment variable one more time | |
| $env:LLVM_SYS_140_PREFIX = $env:LLVM_SYS_140_PREFIX | |
| # Build and test | |
| make build | |
| make pytest-all | |
| - name: Build and test PECOS (non-Windows) | |
| if: runner.os != 'Windows' | |
| run: | | |
| # On macOS, set up minimal environment for matplotlib compatibility | |
| if [[ "${{ runner.os }}" == "macOS" ]]; then | |
| # Set matplotlib backend to avoid GUI issues | |
| export MPLBACKEND=Agg | |
| export MATPLOTLIB_INTERACTIVE=false | |
| # Try to fix matplotlib segfault by limiting threading | |
| export OPENBLAS_NUM_THREADS=1 | |
| export MKL_NUM_THREADS=1 | |
| export NUMEXPR_NUM_THREADS=1 | |
| export OMP_NUM_THREADS=1 | |
| # Force matplotlib to use bundled libraries instead of system ones | |
| export MPLCONFIGDIR=$PWD/.matplotlib | |
| mkdir -p $MPLCONFIGDIR | |
| # CRITICAL: Prevent Homebrew library paths from being used during linking | |
| # This fixes the "@rpath/libunwind.1.dylib" runtime error on macOS | |
| # Reference: https://github.com/rust-lang/rust/issues/135372 | |
| # | |
| # The issue: When LIBRARY_PATH or similar environment variables include | |
| # Homebrew paths (like /usr/local/lib or /opt/homebrew/lib), the linker | |
| # finds Homebrew's libunwind and creates @rpath references that fail at runtime. | |
| # | |
| # The solution: Clear these variables so the linker ONLY uses system libraries | |
| # from /usr/lib (which are in the dyld shared cache). | |
| unset LIBRARY_PATH | |
| unset LD_LIBRARY_PATH | |
| unset DYLD_LIBRARY_PATH | |
| unset DYLD_FALLBACK_LIBRARY_PATH | |
| # Also prevent pkg-config from finding Homebrew packages | |
| unset PKG_CONFIG_PATH | |
| # Set explicit library path to ONLY include system directories | |
| export LIBRARY_PATH=/usr/lib | |
| # DEBUG: Show what environment variables are set | |
| echo "=== Environment variables that affect linking ===" | |
| env | grep -E "LIBRARY|DYLD|PKG_CONFIG|HOMEBREW|PATH" | sort | |
| # DEBUG: Check what libraries are in Homebrew paths | |
| echo "=== Checking for libunwind in Homebrew locations ===" | |
| ls -la /usr/local/lib/libunwind* 2>&1 || echo "No libunwind in /usr/local/lib" | |
| ls -la /opt/homebrew/lib/libunwind* 2>&1 || echo "No libunwind in /opt/homebrew/lib" | |
| # DEBUG: Check system libunwind | |
| echo "=== System libunwind ===" | |
| ls -la /usr/lib/system/libunwind* 2>&1 || echo "No libunwind in /usr/lib/system" | |
| ls -la /usr/lib/libunwind* 2>&1 || echo "No libunwind in /usr/lib" | |
| # DEBUG: Check if LLVM itself has libunwind references | |
| echo "=== Checking LLVM libraries for libunwind references ===" | |
| echo "LLVM dylib files:" | |
| ls -lh /tmp/llvm/lib/*.dylib 2>&1 | head -10 || echo "No dylib files found" | |
| # Check ALL LLVM dylibs for libunwind references | |
| echo "" | |
| echo "Checking each LLVM library for libunwind:" | |
| for lib in /tmp/llvm/lib/*.dylib; do | |
| if [ -f "$lib" ]; then | |
| libname=$(basename "$lib") | |
| if otool -L "$lib" 2>/dev/null | grep -q "libunwind"; then | |
| echo " [WARNING] $libname HAS libunwind reference:" | |
| otool -L "$lib" | grep libunwind | |
| fi | |
| fi | |
| done | |
| echo "Done checking LLVM libraries" | |
| # Check what libc++ the system has | |
| echo "" | |
| echo "=== System C++ library ===" | |
| ls -lh /usr/lib/libc++* 2>&1 | head -5 || echo "No libc++ in /usr/lib" | |
| # Check if clang has any default library search paths configured | |
| echo "" | |
| echo "=== Clang default library search paths ===" | |
| /tmp/llvm/bin/clang -Xlinker -v 2>&1 | grep -A 20 "Library search" || echo "Could not get search paths" | |
| echo "" | |
| echo "=== RUSTFLAGS: $RUSTFLAGS ===" | |
| echo "=== LIBRARY_PATH: $LIBRARY_PATH ===" | |
| fi | |
| # Build with verbose cargo output to see linker commands | |
| echo "" | |
| echo "=== Starting build ===" | |
| if [[ "${{ runner.os }}" == "macOS" ]]; then | |
| # Enable verbose cargo output to see full linker commands | |
| CARGO_LOG=cargo::core::compiler::fingerprint=info make build 2>&1 | tee /tmp/build.log | |
| else | |
| make build 2>&1 | tee /tmp/build.log | |
| fi | |
| # After build, check if the extension has the bad reference | |
| if [[ "${{ runner.os }}" == "macOS" ]]; then | |
| echo "" | |
| echo "=== Checking built extension module ===" | |
| EXT_MODULE=$(find python/pecos-rslib/src/pecos_rslib -name "_pecos_rslib*.so" | head -1) | |
| if [ -n "$EXT_MODULE" ]; then | |
| echo "Found: $EXT_MODULE" | |
| echo "" | |
| echo "=== ALL dependencies of extension module ===" | |
| otool -L "$EXT_MODULE" | |
| echo "" | |
| echo "=== Checking for problematic @rpath reference ===" | |
| if otool -L "$EXT_MODULE" | grep "@rpath/libunwind"; then | |
| echo "❌ ERROR: Still has @rpath/libunwind reference!" | |
| echo "" | |
| echo "=== Let's trace where this comes from ===" | |
| echo "Dependencies that might be the source:" | |
| otool -L "$EXT_MODULE" | grep -v "$EXT_MODULE" | grep "\.dylib" | while read -r line; do | |
| dep=$(echo "$line" | awk '{print $1}') | |
| if [ -f "$dep" ] || [ -L "$dep" ]; then | |
| echo "" | |
| echo "Checking $dep:" | |
| otool -L "$dep" 2>/dev/null | grep -i unwind || echo " No libunwind reference" | |
| fi | |
| done | |
| echo "" | |
| echo "=== Last 100 lines of build log (looking for linking commands) ===" | |
| tail -100 /tmp/build.log | grep -B 2 -A 2 "linking\|rustc.*cdylib\|-L" | |
| exit 1 | |
| else | |
| echo "[OK] No @rpath/libunwind reference found" | |
| fi | |
| else | |
| echo "[WARNING] Could not find extension module to check" | |
| fi | |
| fi | |
| make pytest-all | |
| - name: Run linting | |
| run: | | |
| # Run all linting checks | |
| make lint # Rust checks + Python pre-commit |