chore: use pixi for codegen environment in ci (#137)

Co-authored-by: Niels Pardon <[email protected]>

Author: tokoko

.devcontainer/Dockerfile

@@ -1,13 +1,5 @@
 FROM mcr.microsoft.com/vscode/devcontainers/python:3.13
 USER vscode
-RUN curl -s "https://get.sdkman.io" | bash
-RUN bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && \
-    sdk install java 25-graalce"
-RUN mkdir -p ~/lib && cd ~/lib && curl -L -O http://www.antlr.org/download/antlr-4.13.2-complete.jar
-ENV ANTLR_JAR="~/lib/antlr-4.13.2-complete.jar"
-RUN cd ~ && curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v33.2/protoc-33.2-linux-x86_64.zip && \
-    unzip protoc-33.2-linux-x86_64.zip -d ~/.local && \
-    rm protoc-33.2-linux-x86_64.zip
-RUN curl -sSL "https://github.com/bufbuild/buf/releases/download/v1.58.0/buf-$(uname -s)-$(uname -m)" -o ~/.local/bin/buf && chmod +x ~/.local/bin/buf
 RUN curl -LsSf https://astral.sh/uv/install.sh | sh
-USER root
+RUN curl -fsSL https://pixi.sh/install.sh | sh
+ENV PATH=~/.pixi/bin:$PATH

.devcontainer/devcontainer.json

@@ -14,7 +14,7 @@
 	// "forwardPorts": [],
 
 	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "uv venv --clear && uv sync --group gen_proto",
+	"postCreateCommand": "uv venv --clear && uv sync",
 
 	// Configure tool-specific properties.
 	"customizations": {

.gitattributes

@@ -1,2 +1,4 @@
 src/substrait/gen/** linguist-generated=true
 src/substrait/extensions/** linguist-generated=true
+# SCM syntax highlighting & preventing 3-way merges
+pixi.lock merge=binary linguist-language=YAML linguist-generated=true -diff

.github/workflows/codegen-check.yml

@@ -16,25 +16,17 @@ jobs:
         with:
           submodules: recursive
 
-      - name: Run code generation in devcontainer
-        uses: devcontainers/[email protected]
+      - uses: prefix-dev/[email protected]
         with:
-          runCmd: |
-            # fetch submodule tags since actions/checkout does not
-            git submodule foreach 'git fetch --unshallow || true'
-            # Run all code generation steps
-            make codegen
+          pixi-version: v0.62.2
+
+      - name: Run code generation
+        run: |
+          # fetch submodule tags since actions/checkout does not
+          git submodule foreach 'git fetch --unshallow || true'
+          # Run all code generation steps
+          pixi run codegen
 
       - name: Check for uncommitted changes
         run: |
-          # Check for diffs, ignoring timestamp lines
-          if ! git diff --quiet --exit-code src/substrait/gen/; then
-            echo "Code generation produced changes. Generated code is out of sync!"
-            echo ""
-            git diff src/substrait/gen/
-            echo ""
-            echo "To fix this, run:"
-            echo "  make codegen"
-            echo "Then commit the changes."
-            exit 1
-          fi
+          pixi run check-codegen

.github/workflows/ruff.yml

@@ -15,20 +15,18 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python: ["3.10"]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v6
         with:
           submodules: recursive
-      - name: Install uv with python
-        uses: astral-sh/setup-uv@v7
+      - uses: prefix-dev/[email protected]
         with:
-          python-version: ${{ matrix.python }}
+          pixi-version: v0.62.2
       - name: Run ruff linter
         run: |
-          uvx [email protected] check
+          pixi run lint
       - name: Run ruff formatter
         run: |
-          uvx [email protected] format --check
+          pixi run format --check

.gitignore

@@ -142,3 +142,6 @@ src/substrait/_version.py
 .directory
 .gdb_history
 .DS_Store
+# pixi environments
+.pixi/*
+!.pixi/config.toml

CONTRIBUTING.md

@@ -13,58 +13,61 @@ uv sync --extra test
 ```
 
 ## Update the substrait submodule locally
+You can run update-submodule task to pull in latest substrait from upstream.
+```
+pixi run update-submodule <version>
+```
+
+Or you can update submodule and run code generation at the same time with
+```
+pixi run update-substrait
+```
+
 This might be necessary if you are updating an existing checkout.
 ```
 git submodule sync --recursive
 git submodule update --init --recursive
 ```
 
-
 # Code generation
 
-You can run the full code generation using the following command or use the individual commands to selectively regenerate the generated code. This does not update the Substrait Git submodule.
+You can run the full code generation using the following command or use the individual commands to selectively regenerate the generated code. This does not update the Substrait Git submodule. You can use pixi environment defined in pyproject.toml which contains all dependencies needed for code generation.
 
 ```
-make codegen
+pixi run codegen
 ```
 
 ## Protobuf stubs
 
-Run the upgrade script to upgrade the submodule and regenerate the protobuf stubs.
-
-```
-uv run --group gen_proto ./update_proto.sh <version>
-```
-
-Or run the proto codegen without updating the Substrait Git submodule:
+Run the proto codegen only:
 
 ```
-make codegen-proto
+pixi run codegen-proto
 ```
 
 ## Antlr grammar
 
 Substrait uses antlr grammar to derive output types of extension functions. Make sure java is installed and ANTLR_JAR environment variable is set. Take a look at .devcontainer/Dockerfile for example setup.
 
 ```
-make antlr
+pixi run antlr
 ```
 
 ## Extensions stubs
 
 Substrait uses jsonschema to describe the data model for extension files.
 
 ```
-make codegen-extensions
+pixi run codegen-extensions
 ```
 
 # Lint & Format
 
-Run the following make commands to lint and format with ruff.
+Run the following pixi tasks to lint and format with ruff.
 
 ```
-make lint
-make format
+pixi run lint
+pixi run format
 ```
 
 # Test

Makefile

@@ -0,0 +1,11 @@
+#!/bin/bash
+if ! git diff --quiet --exit-code src/substrait/gen/; then
+    echo "Code generation produced changes. Generated code is out of sync!"
+    echo ""
+    git diff src/substrait/gen/
+    echo ""
+    echo "To fix this, run:"
+    echo "  pixi run codegen"
+    echo "Then commit the changes."
+    exit 1
+fi