Prebuild #25
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Prebuild | |
| # yamllint disable-line rule:truthy | |
| on: | |
| workflow_dispatch: | |
| permissions: | |
| attestations: write | |
| contents: write | |
| id-token: write | |
| env: | |
| # Disable incremental compilation to avoid overhead. | |
| CARGO_INCREMENTAL: "0" | |
| jobs: | |
| release: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # do NOT load any caches here, we want a clean, freestanding build w/o stateful dependencies! | |
| - name: Set Timestamp | |
| id: timestamp | |
| run: | | |
| ts_iso="$(date -u -Is)" | |
| ts_tag="$(echo "$ts_iso" | sed 's/+00:00//g' | sed 's/:/-/g')" | |
| echo "iso=$ts_iso" | |
| echo "tag=$ts_tag" | |
| echo TIMESTAMP_ISO="$ts_iso" >> "$GITHUB_OUTPUT" | |
| echo TIMESTAMP_TAG="$ts_tag" >> "$GITHUB_OUTPUT" | |
| - name: Checkout | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| fetch-depth: 1 | |
| - name: Free disk space | |
| uses: ./.github/actions/free-disk-space | |
| - name: Install `uv` | |
| uses: astral-sh/setup-uv@803947b9bd8e9f986429fa0c5a41c367cd732b41 # v7 | |
| with: | |
| version-file: "python-tooling/pyproject.toml" | |
| - name: Install `just` | |
| uses: taiki-e/install-action@650c5ca14212efbbf3e580844b04bdccf68dac31 # v2 | |
| with: | |
| tool: just | |
| - name: Install stable toolchain | |
| uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master | |
| with: | |
| toolchain: stable | |
| targets: wasm32-wasip2 | |
| - name: build "add one" example (debug) | |
| run: just guests::rust::build-add-one-debug | |
| - name: build "add one" example (release) | |
| run: just guests::rust::build-add-one-release | |
| - name: build python guest (debug) | |
| run: just guests::python::build-debug | |
| - name: build python guest (release) | |
| run: just guests::python::build-release | |
| # we need unique file names for the release | |
| - name: stage release files | |
| run: | | |
| mkdir out | |
| mv target/wasm32-wasip2/debug/examples/add_one.wasm out/example_add_one.debug.wasm | |
| mv target/wasm32-wasip2/release/examples/add_one.wasm out/example_add_one.release.wasm | |
| mv target/wasm32-wasip2/debug/datafusion_udf_wasm_python.wasm out/datafusion_udf_wasm_python.debug.wasm | |
| mv target/wasm32-wasip2/release/datafusion_udf_wasm_python.wasm out/datafusion_udf_wasm_python.release.wasm | |
| - name: compile WASM to machine code | |
| run: | | |
| targets=( | |
| "aarch64-apple-darwin" | |
| "aarch64-pc-windows-msvc" | |
| "aarch64-unknown-linux-gnu" | |
| "x86_64-apple-darwin" | |
| "x86_64-pc-windows-gnu" | |
| "x86_64-pc-windows-msvc" | |
| "x86_64-unknown-linux-gnu" | |
| ) | |
| for wasm_path in out/*.wasm; do | |
| wasm_basename="$(basename "$wasm_path" .wasm)" | |
| for target in "${targets[@]}"; do | |
| out_file="${wasm_basename}.${target}.elf" | |
| echo "$out_file" | |
| cargo run --bin=compile --features="all-arch" -- "$wasm_path" "out/${out_file}" "$target" | |
| done | |
| done | |
| - name: calculate checksums | |
| run: | | |
| sha256sum * > sha256sum.txt | |
| working-directory: ./out | |
| - name: read artifact versions | |
| id: versions | |
| run: | | |
| cargo_md="$(cargo metadata --format-version=1)" | |
| v_arrow="$(echo "$cargo_md" | jq --raw-output '.packages[] | select(.name == "arrow") | .version')" | |
| v_datafusion="$(echo "$cargo_md" | jq --raw-output '.packages[] | select(.name == "datafusion") | .version')" | |
| v_wasmtime="$(echo "$cargo_md" | jq --raw-output '.packages[] | select(.name == "wasmtime") | .version')" | |
| v_wit="$(cat "wit/world.wit" | grep package | sed -E 's/^[^@]+@([0-9.]+).*/\1/g')" | |
| echo " arrow: $v_arrow" | |
| echo "datafusion: $v_datafusion" | |
| echo " wasmtime: $v_wasmtime" | |
| echo " wit: $v_wit" | |
| echo ARROW="$v_arrow" >> "$GITHUB_OUTPUT" | |
| echo DATAFUSION="$v_datafusion" >> "$GITHUB_OUTPUT" | |
| echo WASMTIME="$v_wasmtime" >> "$GITHUB_OUTPUT" | |
| echo WIT="$v_wit" >> "$GITHUB_OUTPUT" | |
| - name: attestation | |
| id: attestation | |
| uses: actions/attest-build-provenance@96278af6caaf10aea03fd8d33a09a777ca52d62f # v3 | |
| with: | |
| subject-path: out/* | |
| - name: publish release | |
| uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2 | |
| with: | |
| body: | | |
| WASM guest binaries. | |
| # Build Metadata | |
| **Commit:** [`${{ github.sha }}`](${{ github.server_url }}/${{ github.repository }}/tree/${{ github.sha }}) | |
| **Build Timestamp:** `${{ steps.timestamp.outputs.TIMESTAMP_ISO }}` | |
| **Build Attestation:** <${{ steps.attestation.outputs.attestation-url }}> | |
| # WIT Interface | |
| The WIT interface — used between host and guest — is: `${{ steps.versions.outputs.WIT }}` | |
| If host & guest are out of sync, you will get an error that looks like this: | |
| ```text | |
| Context("link WASM components", External(Error { context: "initialize bindings", source: no exported instance named `datafusion-udf-wasm:udf/types@x.y.z` | |
| ``` | |
| # Dependency Versions | |
| These are the versions of the major dependencies: | |
| | Dependency | Version | | |
| | - | - | | |
| | Arrow | `${{ steps.versions.outputs.ARROW }}` | | |
| | DataFusion | `${{ steps.versions.outputs.DATAFUSION }}` | | |
| | Wasmtime | `${{ steps.versions.outputs.WASMTIME }}` | | |
| # WASM Binaries | |
| We build the following targets: | |
| - `example_add_one`: "add one" Rust example UDF | |
| - `datafusion_udf_wasm_python`: Guest for Python-based UDFs, bundles CPython. | |
| Each of them is provided in `debug` and `release` builds. The artifacts have the file extension `.wasm`. | |
| # Machine-Code Binaries | |
| In addition to the WASM binary blobs, we also provide pre-compiled guests for various targets to be used with `wasmtime`. The artifacts have the file extension `.elf`. | |
| **⚠️ Using pre-build binaries requires you to validate the source and trust the binary. Using a binary blindly can lead to remote code execution, memory and data corruption, and all kinds of bad things. You can use the provided build attestation. ⚠️** | |
| # Checksums & Attestation | |
| We provide sha256 checksums for all artifacts in `sha256sum.txt`. You can use that file -- either directly or indirectly via its own checksum -- to safely pin all artifacts of this release. We also provide [build attestation](${{ steps.attestation.outputs.attestation-url }}). | |
| preserve_order: true | |
| files: out/* | |
| name: "WASM Binaries ${{ steps.timestamp.outputs.TIMESTAMP_ISO }}" | |
| tag_name: "wasm-binaries/${{ steps.timestamp.outputs.TIMESTAMP_TAG }}/${{ github.sha }}" | |
| fail_on_unmatched_files: true | |
| target_commitish: "${{ github.sha }}" | |
| make_latest: false |