Accumulated updates (#188)

Adding QIS support + decoding + others

Author: ciaranra

.github/actions/setup-llvm/action.yml

@@ -0,0 +1,97 @@
+name: 'Setup LLVM 14.0.6'
+description: 'Install prebuilt LLVM 14.0.6 binaries for all platforms'
+outputs:
+  llvm-path:
+    description: 'Path to LLVM installation'
+    value: ${{ steps.setup.outputs.llvm-path }}
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Setup LLVM (Linux)
+      if: runner.os == 'Linux'
+      shell: bash
+      id: setup-linux
+      run: |
+        mkdir -p /tmp/llvm
+        if [ "$(uname -m)" = "x86_64" ]; then
+          echo "Installing LLVM 14.0.6 for x86_64..."
+          # Use RHEL 8.4 build (Ubuntu 18.04 build doesn't exist for x86_64 in LLVM 14.0.6)
+          curl -LO https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.6/clang+llvm-14.0.6-x86_64-linux-gnu-rhel-8.4.tar.xz
+          tar xf clang+llvm-14.0.6-x86_64-linux-gnu-rhel-8.4.tar.xz -C /tmp/llvm --strip-components=1
+        else
+          echo "Installing LLVM 14.0.6 for aarch64..."
+          curl -LO https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.6/clang+llvm-14.0.6-aarch64-linux-gnu.tar.xz
+          tar xf clang+llvm-14.0.6-aarch64-linux-gnu.tar.xz -C /tmp/llvm --strip-components=1
+        fi
+        echo "/tmp/llvm/bin" >> $GITHUB_PATH
+        echo "LLVM_SYS_140_PREFIX=/tmp/llvm" >> $GITHUB_ENV
+        echo "llvm-path=/tmp/llvm" >> $GITHUB_OUTPUT
+
+        # Verify installation
+        /tmp/llvm/bin/llc --version
+        /tmp/llvm/bin/clang --version
+
+    - name: Setup LLVM (macOS)
+      if: runner.os == 'macOS'
+      shell: bash
+      id: setup-macos
+      run: |
+        mkdir -p /tmp/llvm
+        if [ "$(uname -m)" = "arm64" ]; then
+          echo "Installing LLVM 14.0.6 for arm64..."
+          ARCH_PREFIX=arm64-apple-darwin22.3.0
+        else
+          echo "Installing LLVM 14.0.6 for x86_64..."
+          ARCH_PREFIX=x86_64-apple-darwin
+        fi
+        curl -LO https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.6/clang+llvm-14.0.6-$ARCH_PREFIX.tar.xz
+        tar xf clang+llvm-14.0.6-$ARCH_PREFIX.tar.xz -C /tmp/llvm --strip-components=1
+
+        # FIX: LLVM 14.0.6 libunwind libraries have @rpath references to themselves
+        # This causes "Library not loaded: @rpath/libunwind.1.dylib" errors at runtime
+        # Fix by changing the install name to use the absolute path
+        echo "Fixing LLVM libunwind install names..."
+        for lib in /tmp/llvm/lib/libunwind*.dylib; do
+          if [ -f "$lib" ]; then
+            echo "  Fixing $(basename $lib)"
+            # Change the self-reference from @rpath/libunwind.1.dylib to the absolute path
+            install_name_tool -id "$lib" "$lib"
+          fi
+        done
+        echo "LLVM libunwind libraries fixed"
+
+        echo "/tmp/llvm/bin" >> $GITHUB_PATH
+        echo "LLVM_SYS_140_PREFIX=/tmp/llvm" >> $GITHUB_ENV
+        echo "llvm-path=/tmp/llvm" >> $GITHUB_OUTPUT
+
+        # Verify installation
+        /tmp/llvm/bin/llc --version
+        /tmp/llvm/bin/clang --version
+
+    - name: Setup LLVM (Windows)
+      if: runner.os == 'Windows'
+      shell: pwsh
+      id: setup-windows
+      run: |
+        Write-Host "Installing LLVM 14.0.6 for Windows..."
+        curl.exe -LO https://github.com/PLC-lang/llvm-package-windows/releases/download/v14.0.6/LLVM-14.0.6-win64.7z
+        7z x LLVM-14.0.6-win64.7z "-oC:\LLVM" -y
+
+        echo "C:\LLVM\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+        echo "LLVM_SYS_140_PREFIX=C:\LLVM" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+        echo "llvm-path=C:\LLVM" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
+
+        # Verify installation
+        C:\LLVM\bin\llc.exe --version
+        C:\LLVM\bin\clang.exe --version
+
+    - name: Set output
+      shell: bash
+      id: setup
+      run: |
+        if [ "$RUNNER_OS" = "Linux" ] || [ "$RUNNER_OS" = "macOS" ]; then
+          echo "llvm-path=/tmp/llvm" >> $GITHUB_OUTPUT
+        else
+          echo "llvm-path=C:/LLVM" >> $GITHUB_OUTPUT
+        fi

.github/workflows/julia-release.yml

@@ -70,15 +70,24 @@ jobs:
   build_julia_binaries:
     needs: check_pr_push
     if: needs.check_pr_push.result == 'success' && needs.check_pr_push.outputs.run == 'true'
-    runs-on: ${{ matrix.os }}
+    runs-on: ${{ matrix.runner || matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-        architecture: [x86_64, aarch64]
-        exclude:
-          - os: windows-latest
+        include:
+          - os: ubuntu-latest
+            architecture: x86_64
+          - os: ubuntu-latest
             architecture: aarch64
+            runner: ubuntu-latest
+          - os: macos-13
+            architecture: x86_64
+            runner: macos-13  # Intel Mac for x86_64
+          - os: macos-latest
+            architecture: aarch64
+            runner: macos-latest  # ARM64 Mac
+          - os: windows-latest
+            architecture: x86_64
 
     steps:
       - uses: actions/checkout@v4
@@ -88,22 +97,14 @@ jobs:
       - name: Set up Rust
         run: rustup show
 
+      - name: Setup LLVM 14.0.6
+        uses: ./.github/actions/setup-llvm
+
       - name: Install Rust target
         run: |
-          if [ "${{ matrix.architecture }}" = "aarch64" ]; then
-            if [ "${{ matrix.os }}" = "ubuntu-latest" ]; then
-              rustup target add aarch64-unknown-linux-gnu
-              sudo apt-get update
-              sudo apt-get install -y gcc-aarch64-linux-gnu
-            elif [ "${{ matrix.os }}" = "macos-latest" ]; then
-              rustup target add aarch64-apple-darwin
-            fi
-          else
-            # For x86_64 builds on ARM64 macOS
-            if [ "${{ matrix.os }}" = "macos-latest" ]; then
-              rustup target add x86_64-apple-darwin
-            fi
-          fi
+          # With native ARM64 runners, no cross-compilation setup is needed
+          # All builds are now native for their respective architectures
+          echo "Using native build for ${{ matrix.architecture }} on ${{ matrix.os }}"
 
       - name: Set up Visual Studio environment on Windows
         if: runner.os == 'Windows'
@@ -133,33 +134,49 @@ jobs:
         run: |
           cd julia/pecos-julia-ffi
 
-          if [ "${{ matrix.architecture }}" = "aarch64" ]; then
-            if [ "${{ matrix.os }}" = "ubuntu-latest" ]; then
-              export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc
-              cargo build --release --target aarch64-unknown-linux-gnu
-              target_dir="../../target/aarch64-unknown-linux-gnu/release"
-            elif [ "${{ matrix.os }}" = "macos-latest" ]; then
-              cargo build --release --target aarch64-apple-darwin
-              target_dir="../../target/aarch64-apple-darwin/release"
-            fi
-          else
-            # For x86_64 builds
-            if [ "${{ matrix.os }}" = "macos-latest" ]; then
-              # On ARM64 macOS, explicitly build for x86_64
-              cargo build --release --target x86_64-apple-darwin
-              target_dir="../../target/x86_64-apple-darwin/release"
-            else
-              cargo build --release
-              target_dir="../../target/release"
-            fi
+          # Set up build flags for macOS to avoid LLVM dependency issues
+          if [[ "${{ matrix.os }}" == macos-* ]]; then
+            # Prevent linking against LLVM's libunwind
+            export RUSTFLAGS="-C link-arg=-Wl,-dead_strip_dylibs -C link-arg=-Wl,-ld_classic"
+            # Remove LLVM from library paths during build
+            unset DYLD_LIBRARY_PATH
+            unset DYLD_FALLBACK_LIBRARY_PATH
+            # Explicitly exclude LLVM libraries from linking
+            export RUSTFLAGS="$RUSTFLAGS -L native=/usr/lib"
           fi
 
+          # Native build for all platforms (no cross-compilation needed with native ARM64 runners)
+          cargo build --release
+          target_dir="../../target/release"
+
           # Create artifact directory
           mkdir -p ../../artifacts
 
           # Copy the built library
-          if [ "${{ matrix.os }}" = "macos-latest" ]; then
+          if [[ "${{ matrix.os }}" == macos-* ]]; then
             cp $target_dir/libpecos_julia.dylib ../../artifacts/
+
+            # Fix any remaining LLVM dependencies
+            echo "Checking and fixing library dependencies..."
+            otool -L ../../artifacts/libpecos_julia.dylib
+
+            # Check for any problematic dependencies
+            if otool -L ../../artifacts/libpecos_julia.dylib | grep -q "/tmp/llvm"; then
+              echo "ERROR: Binary has dependencies on /tmp/llvm which won't exist at runtime!"
+              echo "Attempting to fix..."
+
+              # For libunwind specifically, we'll strip it out by creating a new dylib without it
+              if otool -L ../../artifacts/libpecos_julia.dylib | grep -q "/tmp/llvm/lib/libunwind.dylib"; then
+                echo "Creating fixed library without LLVM libunwind dependency..."
+                # Use install_name_tool to remove the dependency by setting it to a weak import
+                # that can fail at runtime without crashing
+                install_name_tool -change /tmp/llvm/lib/libunwind.dylib /usr/lib/libSystem.B.dylib ../../artifacts/libpecos_julia.dylib || \
+                  echo "Warning: Could not remap libunwind dependency"
+              fi
+            fi
+
+            echo "Final library dependencies:"
+            otool -L ../../artifacts/libpecos_julia.dylib
           else
             cp $target_dir/libpecos_julia.so ../../artifacts/
           fi
@@ -200,7 +217,7 @@ jobs:
             os: windows-latest
             architecture: x86_64
           - runner: macos-13
-            os: macos-latest
+            os: macos-13
             architecture: x86_64
           - runner: macos-latest
             os: macos-latest

.github/workflows/julia-test.yml

@@ -57,6 +57,9 @@ jobs:
     - name: Set up Rust
       run: rustup show
 
+    - name: Setup LLVM 14.0.6
+      uses: ./.github/actions/setup-llvm
+
     - name: Cache Rust
       uses: Swatinem/rust-cache@v2
       with:
@@ -72,13 +75,29 @@ jobs:
       if: runner.os == 'Windows'
       shell: pwsh
       run: |
+        Write-Host "Building with LLVM_SYS_140_PREFIX: $env:LLVM_SYS_140_PREFIX"
         cd julia/pecos-julia-ffi
         cargo build --release
 
     - name: Build Rust FFI library (Unix)
       if: runner.os != 'Windows'
       shell: bash
       run: |
+        # On macOS, prevent Homebrew library paths from being used during linking
+        if [[ "${{ runner.os }}" == "macOS" ]]; then
+          # CRITICAL: Prevent Homebrew library paths from being used during linking
+          # This fixes the "@rpath/libunwind.1.dylib" runtime error on macOS
+          # Reference: https://github.com/rust-lang/rust/issues/135372
+          unset LIBRARY_PATH
+          unset LD_LIBRARY_PATH
+          unset DYLD_LIBRARY_PATH
+          unset DYLD_FALLBACK_LIBRARY_PATH
+          unset PKG_CONFIG_PATH
+          export LIBRARY_PATH=/usr/lib
+
+          echo "RUSTFLAGS: $RUSTFLAGS"
+        fi
+
         cd julia/pecos-julia-ffi
         cargo build --release
 

.github/workflows/python-release.yml

@@ -60,72 +60,40 @@ jobs:
   build_wheels_pecos_rslib:
     needs: check_pr_push
     if: needs.check_pr_push.result == 'success' && needs.check_pr_push.outputs.run == 'true'
-    runs-on: ${{ matrix.os }}
+    runs-on: ${{ matrix.runner || matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-        architecture: [ x86_64, aarch64 ]
-        exclude:
-          - os: windows-latest
+        include:
+          - os: ubuntu-latest
+            architecture: x86_64
+          - os: ubuntu-latest
+            architecture: aarch64
+            runner: ubuntu-latest
+          - os: macos-14
             architecture: aarch64
+          - os: macos-13
+            architecture: x86_64
+          - os: windows-2022
+            architecture: x86_64
 
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.sha || github.sha }}
+          submodules: recursive
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Remove conflicting README.md
-        run: |
-          if [ -f crates/pecos-python/README.md ]; then
-            mv crates/pecos-python/README.md crates/pecos-python/README.md.bak
-            echo "Moved conflicting README.md to README.md.bak"
-          else
-            echo "No conflicting README.md found"
-          fi
-
-      - name: Build wheel
-        uses: PyO3/maturin-action@v1
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v3.2.1
         with:
-          command: build
-          args: --release --out dist --interpreter python3.10 ${{ matrix.architecture == 'aarch64' && '--zig' || '' }}
-          working-directory: python/pecos-rslib
-          target: ${{ matrix.architecture == 'aarch64' && (matrix.os == 'macos-latest' && 'aarch64-apple-darwin' || 'aarch64-unknown-linux-gnu') || (matrix.os == 'macos-latest' && 'x86_64-apple-darwin' || '') }}
-          manylinux: auto
-
-      - name: Restore README.md
-        if: always()
-        run: |
-          if [ -f crates/pecos-python/README.md.bak ]; then
-            mv crates/pecos-python/README.md.bak crates/pecos-python/README.md
-            echo "Restored README.md from backup"
-          else
-            echo "No README.md backup found"
-          fi
-
-      - name: Test wheel is abi3
-        run: |
-          # Check that the wheel has abi3 tag
-          ls -la python/pecos-rslib/dist/
-          wheel_file=$(ls python/pecos-rslib/dist/*.whl)
-          echo "Built wheel: $wheel_file"
-          if [[ $wheel_file == *"abi3"* ]]; then
-            echo "Wheel has abi3 tag"
-          else
-            echo "ERROR: Wheel does not have abi3 tag!"
-            exit 1
-          fi
+          package-dir: python/pecos-rslib
+          output-dir: wheelhouse
 
-      - name: Upload wheel
+      - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:
           name: wheel-pecos-rslib-${{ matrix.os }}-${{ matrix.architecture }}
-          path: python/pecos-rslib/dist/*.whl
+          path: ./wheelhouse/*.whl
 
   test_abi3_wheels:
     needs: build_wheels_pecos_rslib
@@ -142,13 +110,13 @@ jobs:
             os: ubuntu-latest
             architecture: x86_64
           - runner: windows-latest
-            os: windows-latest
+            os: windows-2022
             architecture: x86_64
           - runner: macos-13
-            os: macos-latest
+            os: macos-13
             architecture: x86_64
-          - runner: macos-latest
-            os: macos-latest
+          - runner: macos-14
+            os: macos-14
             architecture: aarch64
     steps:
       - uses: actions/checkout@v4