Merge pull request #197 from cipherstash/benchmarks

feat: pgbench with postgres, pgbouncer and proxy

Author: tobyhede

.github/actions/setup-rust/action.yml

@@ -0,0 +1,22 @@
+name: Setup Test
+description: |
+  This action sets up the environment for running tests in a Rust project.
+  It installs Rust, sets up caching, and installs mise.
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/checkout@v4
+    - name: Install rust
+      shell: /bin/bash -l {0}
+      run: rustup toolchain install stable --profile minimal --no-self-update
+    - name: Setup Rust cache
+      uses: Swatinem/rust-cache@v2
+      with:
+        cache-provider: buildjet
+        cache-all-crates: true
+    - uses: jdx/mise-action@v2
+      with:
+        version: 2025.1.0 # [default: latest] mise version to install
+        install: true # [default: true] run `mise install`
+        cache: true # [default: true] cache mise using GitHub's cache

.github/actions/setup-test/action.yml

@@ -0,0 +1,59 @@
+name: Benchmark Proxy
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+env:
+  CARGO_TERM_COLOR: always
+
+permissions:
+  # deployments permission to deploy GitHub pages website
+  deployments: write
+  # contents permission to update benchmark contents in gh-pages branch
+  contents: write
+
+jobs:
+  benchmark:
+    name: Performance regression check
+    runs-on: buildjet-16vcpu-ubuntu-2204
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-test
+      - run: |
+          mise run postgres:up --extra-args "--detach --wait"
+      - name: Run benchmark
+        working-directory: tests/benchmark
+        env:
+          CS_WORKSPACE_ID: ${{ secrets.CS_WORKSPACE_ID }}
+          CS_CLIENT_ACCESS_KEY: ${{ secrets.CS_CLIENT_ACCESS_KEY }}
+          CS_DEFAULT_KEYSET_ID: ${{ secrets.CS_DEFAULT_KEYSET_ID }}
+          CS_CLIENT_ID: ${{ secrets.CS_CLIENT_ID }}
+          CS_CLIENT_KEY: ${{ secrets.CS_CLIENT_KEY }}
+          RUST_BACKTRACE: "1"
+        run: mise run benchmark:continuous
+      # Download previous benchmark result from cache (if exists)
+      - name: Download previous benchmark data
+        uses: actions/cache@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+      # Run `github-action-benchmark` action
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          # What benchmark tool the output.txt came from
+          tool: 'customSmallerIsBetter'
+          # Where the output from the benchmark tool is stored
+          output-file-path: tests/benchmark/results/output.json
+
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          fail-on-alert: true
+          comment-on-alert: true
+          summary-always: true
+          auto-push: true
+          benchmark-data-dir-path: docs

.github/workflows/benchmark.yml

@@ -1,4 +1,4 @@
-name: test
+name: Test
 on:
   pull_request:
     branches:
@@ -10,25 +10,14 @@ on:
 
 env:
   CARGO_TERM_COLOR: always
+
 jobs:
   test:
     name: Test
     runs-on: buildjet-16vcpu-ubuntu-2204
     steps:
       - uses: actions/checkout@v4
-      - name: Install rust
-        shell: /bin/bash -l {0}
-        run: rustup toolchain install stable --profile minimal --no-self-update
-      - name: Setup Rust cache
-        uses: Swatinem/rust-cache@v2
-        with:
-          cache-provider: buildjet
-          cache-all-crates: true
-      - uses: jdx/mise-action@v2
-        with:
-          version: 2025.1.0 # [default: latest] mise version to install
-          install: true # [default: true] run `mise install`
-          cache: true # [default: true] cache mise using GitHub's cache
+      - uses: ./.github/actions/setup-test
       - run: |
           mise run postgres:up --extra-args "--detach --wait"
       - env:

.github/workflows/test.yml

@@ -18,3 +18,7 @@ rust-toolchain.toml
 
 # credentials for local dev
 .env.proxy.docker
+
+## benchmark result data
+tests/benchmark/results/*.csv
+tests/benchmark/benchmark-*.png

.gitignore

@@ -10,17 +10,29 @@ edition = "2021"
 incremental = true
 debug = true
 
+# [profile.dev.package]# aws-lc-sys.opt-level = 3
+# proc-macro2.opt-level = 3
+# quote.opt-level = 3
+# serde_derive.opt-level = 3
+# sqlparser.opt-level = 3
+# syn.opt-level = 3
+
 [profile.dev.build-override]
 opt-level = 3
 
 [profile.test]
 incremental = true
 debug = true
 
+#  Default release profile (https://doc.rust-lang.org/cargo/reference/profiles.html)
 [profile.release]
 codegen-units = 1
 strip = "symbols"
-lto = true
+
+[profile.profiling]
+inherits = "release"
+strip = "none"
+debug = true
 
 [workspace.dependencies]
 sqlparser = { version = "^0.52", features = ["bigdecimal", "serde"] }

Cargo.toml

@@ -786,6 +786,41 @@ If the proxy is running on a host other than localhost, access on that host.
 | `-h`, `--help`          | Displays this help message                                     | -               |
 
 
+## Benchmarks
+
+Benchmarking is integrated into CI, and can be run locally.
+
+The benchmark uses `pgbench` to compare direct access to PostgreSQL against `pgbouncer` and Proxy.
+Benchmarks are executed in Docker containers, as the focus is on providing a repeatable baseline for performance.
+Your mileage may vary when comparing against a "real-world" production configuration.
+
+The benchmarks use the extended protocol option and cover:
+- the default `pgbench` transaction
+- insert, update & select of plaintext data
+- insert, update & select of encrypted data (CipherStash Proxy only)
+
+The benchmark setup includes the database configuration, but does requires access to a CipherStash account environment.
+
+### Require environment variables
+```
+CS_WORKSPACE_ID
+CS_CLIENT_ACCESS_KEY
+CS_DEFAULT_KEYSET_ID
+CS_CLIENT_ID
+CS_CLIENT_KEY
+```
+
+
+### Running the benchmark
+
+```bash
+cd tests/benchmark
+mise run benchmark
+```
+
+Results are graphed in a file called `benchmark-{YmdHM}.png` where `YmdHM` is a generated timestamp.
+Detailed results are generated in `csv` format and in the `results` directory.
+
 
 ## More info
 

README.md

@@ -111,6 +111,15 @@ mise run test:unit
 mise run test:integration
 """
 
+[tasks."reset"]
+alias = 'r'
+description = "Reset database"
+run = """
+mise run postgres:down
+mise run postgres:up --extra-args "--detach --wait"
+mise run postgres:setup
+"""
+
 [tasks."test:local:unit"]
 alias = 'lu'
 description = "Runs test/s"

mise.toml

@@ -56,7 +56,6 @@ uuid = { version = "1.11.0", features = ["serde", "v4"] }
 x509-parser = "0.17.0"
 vitaminc-protected = "0.1.0-pre2"
 
-
 [dev-dependencies]
 recipher = "0.1.3"
 temp-env = "0.3.6"

packages/cipherstash-proxy/Cargo.toml

@@ -1,6 +1,5 @@
-use bytes::{BufMut, BytesMut};
-
 use super::FrontendCode;
+use bytes::{BufMut, BytesMut};
 
 pub struct Terminate;