Add Linux Local Redirect Mode (#206)

Author: mhils

.github/actions/setup/action.yml

@@ -0,0 +1,26 @@
+name: 'setup'
+description: 'checkout, setup rust and python'
+inputs:
+  rust-version:
+    description: 'Rust version'
+    required: true
+    default: 'stable'
+  extra-targets:
+    description: 'Extra Rust targets'
+  toolchain-args:
+    description: 'Extra args for `rustup toolchain`'
+runs:
+  using: "composite"
+  steps:
+    - uses: mhils/workflows/setup-python@v12  # PyO3 wants recent Python on Windows.
+    - run: rustup toolchain install ${{ inputs.rust-version }} --profile minimal ${{ inputs.toolchain-args }}
+      shell: bash
+    - run: rustup default ${{ inputs.rust-version }}
+      shell: bash
+    - if: inputs.extra-targets
+      run: rustup target add ${{ inputs.extra-targets }}
+      shell: bash
+    - uses: mhils/workflows/rust-cache@v14
+    - if: runner.os == 'Linux'
+      run: cargo install --locked bpf-linker
+      shell: bash

.github/python-version.txt

@@ -1 +1 @@
-3.11
+3.12

.github/scripts/pin-versions.py

@@ -11,6 +11,7 @@
 contents = (
     contents
     .replace(f"mitmproxy_windows", f"mitmproxy_windows=={version}")
+    .replace(f"mitmproxy_linux", f"mitmproxy_linux=={version}")
     .replace(f"mitmproxy_macos", f"mitmproxy_macos=={version}")
 )
 pyproject_toml.write_text(contents)

.github/workflows/autofix.yml

@@ -11,17 +11,18 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  rust_clippy: "1.80"  # MSRV
+  MSRV: "1.80"  # Minimum Supported Rust Version
 
 jobs:
   protobuf:
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: mhils/workflows/checkout@v12
       - run: brew install swift-protobuf
       - run: cargo install protoc-gen-prost
       - run: protoc --proto_path=./src/ipc/ mitmproxy_ipc.proto
           --prost_out=./src/ipc/
+          --prost_opt="bytes=data"
           --swift_out=./mitmproxy-macos/redirector/ipc
       - run: cargo fmt --all
       - uses: autofix-ci/action@ff86a557419858bb967097bfc916833f5647fa8c
@@ -35,18 +36,17 @@ jobs:
           - os: macos-latest
           - os: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - run: rustup toolchain install ${{ env.rust_clippy }} --profile minimal --component rustfmt --component clippy
-      - run: rustup default ${{ env.rust_clippy }}
-      - uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab  # v2.7.5
-        timeout-minutes: 2
-        continue-on-error: true
-      # PyO3 wants recent Python on Windows.
-      - uses: actions/setup-python@v5
+      - uses: mhils/workflows/checkout@v12
+      - uses: ./.github/actions/setup
         with:
-          python-version-file: .github/python-version.txt
-
-      - run: cargo clippy --fix --allow-dirty --workspace
+          rust-version: ${{ env.MSRV }}
+          toolchain-args: --component rustfmt --component clippy
+
+        # We could run clippy on mitmproxy-linux-ebpf with
+        #   cargo +nightly clippy --workspace -- -C panic=abort -Zpanic_abort_tests
+        # but that means we'd use nightly clippy, which may change its behavior (and thus break CI).
+        # So we rather exempt mitmproxy-linux-ebpf from clippy lints.
+      - run: cargo clippy --fix --allow-dirty --workspace --exclude mitmproxy-linux-ebpf
       - run: cargo fmt --all
       - run: git checkout src/ipc/mitmproxy_ipc.rs
 

.github/workflows/ci.yml

@@ -22,25 +22,27 @@ jobs:
       matrix:
         include:
           - os: windows-latest
-            rust: "1.80"  # MSRV
-            args: --exclude macos-certificate-truster
+            rust: "1.80"  # MSRV - can't use variables here.
+            args: --exclude mitmproxy-linux-ebpf
           - os: macos-latest
-            rust: "1.80"
-            args: --exclude windows-redirector
+            rust: "1.80"  # MSRV - can't use variables here.
+            args: --exclude mitmproxy-linux-ebpf
+          - os: ubuntu-22.04
+            rust: "1.80"  # MSRV - can't use variables here.
+            args: --exclude mitmproxy-linux-ebpf
           - os: ubuntu-latest
             rust: stable
-            args: --exclude windows-redirector --exclude macos-certificate-truster
+            args: --exclude mitmproxy-linux-ebpf
+          - os: ubuntu-latest  # old Ubuntu to test eBPF verifier compatibility
+            rust: nightly
+            args: --package mitmproxy-linux-ebpf
+    env:
+      RUSTFLAGS: ${{ matrix.rust == 'nightly' && '-Zpanic_abort_tests -C panic=abort' || '' }}
     steps:
-      - uses: actions/checkout@v4
-      - name: Set up Rust toolchain
-        run: rustup toolchain install ${{ matrix.rust }} --profile minimal
-      - uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab  # v2.7.5
-        timeout-minutes: 2
-        continue-on-error: true
-      # PyO3 wants recent Python on Windows.
-      - uses: actions/setup-python@v5
+      - uses: mhils/workflows/checkout@v12
+      - uses: ./.github/actions/setup
         with:
-          python-version-file: .github/python-version.txt
+          rust-version: ${{ matrix.rust }}
 
       - name: Run "cargo check"
         # the action-rs/cargo action adds inline annotations for "cargo check" output
@@ -49,13 +51,19 @@ jobs:
           toolchain: ${{ matrix.rust }}
           command: check
           args: --workspace --verbose ${{ matrix.args }}
-      - name: Run "cargo test"
+      - if: matrix.rust != 'nightly'  # XXX: weird errors here
+        name: Run "cargo test"
         # the action-rs/cargo action adds inline annotations for "cargo test" output
         uses: actions-rs/cargo@9e120dd99b0fbad1c065f686657e914e76bd7b72
         with:
           toolchain: ${{ matrix.rust }}
           command: test
           args: --workspace --verbose ${{ matrix.args }}
+      - if: matrix.os == 'ubuntu-22.04'  # Test that eBPF loads
+        run: cargo test --features root-tests
+        working-directory: mitmproxy-linux
+        env:
+          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: sudo -E
 
   build:
     strategy:
@@ -69,33 +77,25 @@ jobs:
           - name: linux-arm64
             os: ubuntu-latest
             target: aarch64-unknown-linux-gnu
-            args: --compatibility manylinux2014 --zig --target aarch64-unknown-linux-gnu
+            args: --compatibility manylinux2014 --zig --target aarch64-unknown-linux-gnu -i 3.12
           - name: macos-universal
             os: macos-latest
             target: aarch64-apple-darwin x86_64-apple-darwin
             args: --target universal2-apple-darwin
     runs-on: ${{ matrix.os }}
     name: build mitmproxy-rs (${{ matrix.name }})
     steps:
-      - uses: actions/checkout@v4
-      - run: rustup toolchain install stable --profile minimal
-      - run: rustup default stable
-      - if: matrix.target
-        run: rustup target add ${{ matrix.target }}
-      - run: rustup show
-      - uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab  # v2.7.5
-        timeout-minutes: 2
-        continue-on-error: true
-      - uses: actions/setup-python@v5
+      - uses: mhils/workflows/checkout@v12
+      - uses: ./.github/actions/setup
         with:
-          python-version-file: .github/python-version.txt
+          extra-targets: ${{ matrix.target }}
 
       - if: runner.os == 'Linux'
         name: Install maturin[zig] from PyPI
-        uses: install-pinned/maturin-with-zig@4804d730717f28b7565e71e68e5c9fe8c4f9b089
+        uses: install-pinned/maturin-with-zig@68c027568b7d08df7bc3c52476ae28d1d2d787f5
       - if: runner.os != 'Linux'
         name: Install maturin from PyPI
-        uses: install-pinned/maturin@c8c3a8f7f2b2ecf1728c96824734c6c6afe7e3e8
+        uses: install-pinned/maturin@b1e3f698dbd19f284d4363cb361f75b2fa04679c
 
       - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
         run: python .github/scripts/pin-versions.py
@@ -106,7 +106,7 @@ jobs:
       # ensure that sdist is building.
       # We do this here instead of a separate job because we don't want to wait for the entire matrix.
       - if: contains(matrix.args, 'sdist')
-        run: pip install target/wheels/*.tar.gz
+        run: pip install --no-dependencies target/wheels/*.tar.gz
 
       - uses: actions/upload-artifact@v4
         with:
@@ -116,7 +116,7 @@ jobs:
   build-macos-app:
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: mhils/workflows/checkout@v12
       - uses: actions/cache@v4
         id: cache-app
         with:
@@ -137,62 +137,96 @@ jobs:
           name: macos-app
           path: mitmproxy-macos/redirector/dist/
 
-  build-os-wheels:
-    needs: build-macos-app
-    strategy:
-      matrix:
-        include:
-          - os: windows
-            build-rust: --package windows-redirector
-          - os: macos
-            build-rust: --package macos-certificate-truster
-    runs-on: ${{ matrix.os }}-latest
-    name: build mitmproxy-${{ matrix.os }}
+  build-windows-wheel:
+    runs-on: windows-latest
+    name: build mitmproxy-windows
     steps:
-      - uses: actions/checkout@v4
-
-      # Build Rust
-      - uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab  # v2.7.5
-        timeout-minutes: 2
-        continue-on-error: true
-      - run: rustup toolchain install stable --profile minimal
-      - run: rustup default stable
-      - if: runner.os == 'macOS'
-        run: rustup target add aarch64-apple-darwin x86_64-apple-darwin
-      - if: runner.os == 'macOS'
-        run: |
-          cargo build --release ${{ matrix.build-rust }} --target x86_64-apple-darwin
-          cargo build --release ${{ matrix.build-rust }} --target aarch64-apple-darwin
-          lipo -create -output target/release/macos-certificate-truster target/x86_64-apple-darwin/release/macos-certificate-truster target/aarch64-apple-darwin/release/macos-certificate-truster
-      - if: runner.os != 'macOS'
-        run: cargo build --release ${{ matrix.build-rust }}
+      - uses: mhils/workflows/checkout@v12
+      - uses: ./.github/actions/setup
+      - uses: install-pinned/build@aa7fb973fec4a5593736c5dc25b322120ca41a98
+
+      - run: cargo build --release --package windows-redirector
+
+      - run: python -m build --wheel ./mitmproxy-windows --outdir target/wheels/
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-os-windows
+          path: target/wheels
 
+  build-macos-wheel:
+    name: build mitmproxy-macos
+    needs: build-macos-app
+    runs-on: macos-latest
+    steps:
+      - uses: mhils/workflows/checkout@v12
+      - uses: ./.github/actions/setup
+        with:
+          extra-targets: aarch64-apple-darwin x86_64-apple-darwin
+      - uses: install-pinned/build@aa7fb973fec4a5593736c5dc25b322120ca41a98
 
-      # Download macOS app
-      - if: runner.os == 'macOS'
-        uses: actions/download-artifact@v4
+      - run: |
+          cargo build --release --package macos-certificate-truster --target x86_64-apple-darwin
+          cargo build --release --package macos-certificate-truster --target aarch64-apple-darwin
+          lipo -create -output target/release/macos-certificate-truster target/x86_64-apple-darwin/release/macos-certificate-truster target/aarch64-apple-darwin/release/macos-certificate-truster
+      - uses: actions/download-artifact@v4
         with:
           name: macos-app
           path: mitmproxy-macos/redirector/dist/
 
-      # Build & upload wheel
-      - uses: actions/setup-python@v5
+      - run: python -m build --wheel ./mitmproxy-macos --outdir target/wheels/
+      - uses: actions/upload-artifact@v4
         with:
-          python-version-file: .github/python-version.txt
-      - name: Install build from PyPI
-        uses: install-pinned/build@67059894d5ef2398af490c98fa8af8542d7b7008
-      - run: python -m build --wheel ./mitmproxy-${{ matrix.os }} --outdir target/wheels/
+          name: wheels-os-macos
+          path: target/wheels
+
+  build-linux-wheel:
+    name: build mitmproxy-${{ matrix.name }}
+    strategy:
+      matrix:
+        include:
+          - name: linux-x86_64
+            args: --compatibility manylinux2014 --zig --sdist
+          - name: linux-arm64
+            target: aarch64-unknown-linux-gnu
+            args: --compatibility manylinux2014 --zig --target aarch64-unknown-linux-gnu -i 3.12
+    runs-on: ubuntu-latest
+    steps:
+      - uses: mhils/workflows/checkout@v12
+      - uses: ./.github/actions/setup
+        with:
+          extra-targets: ${{ matrix.target }}
+      - name: Install maturin[zig] from PyPI
+        uses: install-pinned/maturin-with-zig@68c027568b7d08df7bc3c52476ae28d1d2d787f5
+
+      - run: maturin build --release ${{ matrix.args }}
+        working-directory: ./mitmproxy-linux
+
       - uses: actions/upload-artifact@v4
         with:
-          name: wheels-os-${{ runner.os }}
+          name: wheels-os-${{ matrix.name }}
+          path: target/wheels
+
+  test-linux-wheel-sdist:
+    needs: build-linux-wheel
+    runs-on: ubuntu-latest
+    steps:
+      - uses: mhils/workflows/checkout@v12
+      - uses: ./.github/actions/setup
+      - uses: actions/download-artifact@v4
+        with:
+          name: wheels-os-linux-x86_64
           path: target/wheels
+      - run: pip install --no-dependencies target/wheels/*.tar.gz
 
   check:
     if: always()
     needs:
       - test
+      - test-linux-wheel-sdist
       - build
-      - build-os-wheels
+      - build-windows-wheel
+      - build-linux-wheel
+      - build-macos-wheel
     uses: mhils/workflows/.github/workflows/alls-green.yml@main
     with:
       jobs: ${{ toJSON(needs) }}

.github/workflows/docs.yml

@@ -19,20 +19,16 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab  # v2.7.5
-        timeout-minutes: 2
-        continue-on-error: true
-      - name: Install maturin[zig] from PyPI
-        uses: install-pinned/maturin-with-zig@4804d730717f28b7565e71e68e5c9fe8c4f9b089
-      - name: Install mypy from PyPI
-        uses: install-pinned/mypy@acfb567eb55a86c12a58ca1d28cd3e183e6a6d63
-      - name: Install pdoc from PyPI
-        uses: install-pinned/pdoc@2e3e48c4027dad61fb97866e009e1e6b7da38209
+      # TODO: This should ideally just reuse the main CI artifacts.
+      - uses: mhils/workflows/checkout@v12
+      - uses: ./.github/actions/setup
+      - uses: install-pinned/maturin-with-zig@68c027568b7d08df7bc3c52476ae28d1d2d787f5
+      - uses: install-pinned/mypy@2b552bed479e3f7065314667b670f7303619e989
+      - uses: install-pinned/pdoc@fd9469ecb06f32b7012e07e449ce98b217bf1189
 
       - run: maturin build
         working-directory: ./mitmproxy-rs
-      - run: pip install --no-index --find-links target/wheels/ mitmproxy_rs
+      - run: pip install --no-index --no-dependencies --find-links target/wheels/ mitmproxy_rs
 
       - run: stubtest --allowlist mitmproxy-rs/stubtest-allowlist.txt --mypy-config-file mitmproxy-rs/pyproject.toml mitmproxy_rs
 

CONTRIBUTING.md

@@ -24,7 +24,7 @@ re-run `maturin develop` and restart mitmproxy** for changes to apply.
 If you've followed the procedure above, you can run the basic test suite as follows:
 
 ```shell
-cargo test --workspace
+cargo test
 ```
 
 Please ensure that all patches are accompanied by matching changes in the test suite.