chore: remove dependency on protoletariat and update protobuf (#138)

Protoletariat is using an outdated version of protobuf and does not seem
to be actively maintained.

By changing the way we generate the Python code from the proto files we
can
remove the dependency on protoletariat and update to the latest protobuf
version.

Additionally, this adds the code formatting and lint fixing Make targets
to the
codegen target and fixes some formatting issues that were introduced in
a prior
PR.

---------

Signed-off-by: Niels Pardon <[email protected]>

Author: nielspardon

.devcontainer/Dockerfile

@@ -5,10 +5,9 @@ RUN bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && \
     sdk install java 25-graalce"
 RUN mkdir -p ~/lib && cd ~/lib && curl -L -O http://www.antlr.org/download/antlr-4.13.2-complete.jar
 ENV ANTLR_JAR="~/lib/antlr-4.13.2-complete.jar"
-# protoc 29.5 is the last version with protobuf python v5 which is compatible with protoletariat v3
-RUN cd ~ && curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v29.5/protoc-29.5-linux-x86_64.zip && \
-    unzip protoc-29.5-linux-x86_64.zip -d ~/.local && \
-    rm protoc-29.5-linux-x86_64.zip
+RUN cd ~ && curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v33.2/protoc-33.2-linux-x86_64.zip && \
+    unzip protoc-33.2-linux-x86_64.zip -d ~/.local && \
+    rm protoc-33.2-linux-x86_64.zip
 RUN curl -sSL "https://github.com/bufbuild/buf/releases/download/v1.58.0/buf-$(uname -s)-$(uname -m)" -o ~/.local/bin/buf && chmod +x ~/.local/bin/buf
 RUN curl -LsSf https://astral.sh/uv/install.sh | sh
 USER root

.github/workflows/ruff.yml

@@ -31,4 +31,4 @@ jobs:
           uvx [email protected] check
       - name: Run ruff formatter
         run: |
-          uvx [email protected] format
+          uvx [email protected] format --check

buf.gen.yaml

@@ -1,6 +1,6 @@
 version: v2
 plugins:
-  - protoc_builtin: python
-    out: src/substrait/gen
-  - remote: buf.build/community/nipunn1313-mypy:v3.6.0
-    out: src/substrait/gen
+  - remote: buf.build/protocolbuffers/python:v33.2
+    out: src
+  - remote: buf.build/community/nipunn1313-mypy:v4.0.0
+    out: src

gen_proto.sh

@@ -2,23 +2,23 @@
 
 set -eou pipefail
 
-namespace=proto
+namespace=substrait.gen.proto
 submodule_dir=./third_party/substrait
 src_dir="$submodule_dir"/proto
 tmp_dir=./buf_work_dir
-dest_dir=./src/substrait/gen
+dest_dir=./src
 extension_dir=./src/substrait/extensions
 
 # Prefix the protobuf files with a unique configuration to prevent namespace conflicts
 # with other substrait packages. Save output to the work dir.
 python "$submodule_dir"/tools/proto_prefix.py "$tmp_dir" "$namespace" "$src_dir"
 
 # Remove the old python protobuf files
-rm -rf "$dest_dir/proto"
+rm -rf "$dest_dir/substrait/gen/proto"
 
 # Generate the new python protobuf files
 buf generate
-uv run protol --in-place --create-package --python-out "$dest_dir" buf
+find "$dest_dir/substrait/gen" -type d -exec touch {}/__init__.py \;
 
 # Remove the old extension files
 rm -rf "$extension_dir"

pyproject.toml

@@ -5,7 +5,7 @@ authors = [{name = "Substrait contributors", email = "[email protected]
 license = {text = "Apache-2.0"}
 readme = "README.md"
 requires-python = ">=3.10,<3.14"
-dependencies = ["protobuf >=3.19.1,<6"]
+dependencies = ["protobuf >=6.33,<7"]
 dynamic = ["version"]
 
 [tool.setuptools_scm]
@@ -17,7 +17,7 @@ sql = ["sqloxide", "deepdiff"]
 
 [dependency-groups]
 dev = ["pytest >= 7.0.0", "antlr4-python3-runtime", "pyyaml", "sqloxide", "deepdiff", "duckdb<=1.2.2", "datafusion"]
-gen_proto = ["protobuf >=3.19.1,<6", "protoletariat >= 3.0.0"]
+gen_proto = ["protobuf >=6.33,<7"]
 gen_extensions = ["datamodel-code-generator"]
 
 [tool.pytest.ini_options]

src/substrait/builders/type.py

@@ -223,7 +223,6 @@ def precision_timestamp_tz(precision: int, nullable=True) -> stt.Type:
     )
 
 
-
 def struct(types: Iterable[stt.Type], nullable=True) -> stt.Type:
     return stt.Type(
         struct=stt.Type.Struct(