Add performance tests #39
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark Workflow | |
| on: | |
| # https://github.com/mxschmitt/action-tmate?tab=readme-ov-file#manually-triggered-debug | |
| workflow_dispatch: | |
| inputs: | |
| debug_enabled: | |
| description: 'Enable SSH access (⚠️ Security Risk - read workflow comments)' | |
| required: false | |
| default: false | |
| type: boolean | |
| rate: | |
| description: 'Requests per second (use "max" for maximum throughput)' | |
| required: false | |
| default: 'max' | |
| type: string | |
| duration: | |
| description: 'Duration (e.g., "30s", "1m", "90s")' | |
| required: false | |
| default: '30s' | |
| type: string | |
| request_timeout: | |
| description: 'Request timeout (e.g., "60s", "1m", "90s")' | |
| required: false | |
| default: '60s' | |
| type: string | |
| connections: | |
| description: 'Concurrent connections/virtual users' | |
| required: false | |
| default: 10 | |
| type: number | |
| max_connections: | |
| description: 'Maximum connections/virtual users' | |
| required: false | |
| type: number | |
| web_concurrency: | |
| description: 'Number of Puma worker processes' | |
| required: false | |
| default: 4 | |
| type: number | |
| rails_max_threads: | |
| description: 'Maximum number of Puma threads' | |
| required: false | |
| default: 3 | |
| type: number | |
| rails_min_threads: | |
| description: 'Minimum number of Puma threads (same as maximum if not set)' | |
| required: false | |
| type: number | |
| tools: | |
| description: 'Comma-separated list of tools to run' | |
| required: false | |
| default: 'fortio,vegeta,k6' | |
| type: string | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| env: | |
| FORTIO_VERSION: "1.73.0" | |
| K6_VERSION: "1.3.0" | |
| VEGETA_VERSION: "12.13.0" | |
| # Benchmark parameters | |
| RATE: ${{ github.event.inputs.rate || 'max' }} | |
| DURATION: ${{ github.event.inputs.duration || '30s' }} | |
| REQUEST_TIMEOUT: ${{ github.event.inputs.request_timeout || '60s' }} | |
| CONNECTIONS: ${{ github.event.inputs.connections || 10 }} | |
| MAX_CONNECTIONS: ${{ github.event.inputs.max_connections || github.event.inputs.connections || 10 }} | |
| WEB_CONCURRENCY: ${{ github.event.inputs.web_concurrency || 4 }} | |
| RAILS_MAX_THREADS: ${{ github.event.inputs.rails_max_threads || 3 }} | |
| RAILS_MIN_THREADS: ${{ github.event.inputs.rails_min_threads || github.event.inputs.rails_max_threads || 3 }} | |
| TOOLS: ${{ github.event.inputs.tools || 'fortio,vegeta,k6' }} | |
| jobs: | |
| benchmark: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # ============================================ | |
| # STEP 1: CHECKOUT CODE | |
| # ============================================ | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| # ============================================ | |
| # STEP 2: OPTIONAL SSH ACCESS | |
| # ============================================ | |
| # NOTE: Interactive confirmation is not possible in GitHub Actions. | |
| # As a secure workaround, SSH access is gated by the workflow_dispatch | |
| # input variable 'debug_enabled' which defaults to false. | |
| # Users must explicitly set this to true to enable SSH. | |
| - name: SSH Warning | |
| if: ${{ github.event.inputs.debug_enabled == true || github.event.inputs.debug_enabled == 'true' }} | |
| run: | | |
| echo "⚠️ ⚠️ ⚠️ SSH ACCESS ENABLED ⚠️ ⚠️ ⚠️" | |
| echo "" | |
| echo "SECURITY NOTICE:" | |
| echo " - SSH access exposes your GitHub Actions runner" | |
| echo " - Only proceed if you understand and accept the risks" | |
| echo " - Do NOT store secrets or sensitive data on the runner" | |
| echo " - Access is limited to the workflow initiator only" | |
| echo " - The session will remain open until manually terminated" | |
| echo "" | |
| echo "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️" | |
| - name: Setup SSH access (if enabled) | |
| if: ${{ github.event.inputs.debug_enabled == true || github.event.inputs.debug_enabled == 'true' }} | |
| uses: mxschmitt/action-tmate@v3 | |
| with: | |
| detached: true | |
| limit-access-to-actor: true # Only workflow trigger can access | |
| # ============================================ | |
| # STEP 3: INSTALL BENCHMARKING TOOLS | |
| # ============================================ | |
| - name: Add tools directory to PATH | |
| run: | | |
| mkdir -p ~/bin | |
| echo "$HOME/bin" >> $GITHUB_PATH | |
| - name: Cache Fortio binary | |
| id: cache-fortio | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/bin/fortio | |
| key: fortio-${{ runner.os }}-${{ runner.arch }}-${{ env.FORTIO_VERSION }} | |
| - name: Install Fortio | |
| if: steps.cache-fortio.outputs.cache-hit != 'true' | |
| run: | | |
| echo "📦 Installing Fortio v${FORTIO_VERSION}" | |
| # Download and extract fortio binary | |
| wget -q https://github.com/fortio/fortio/releases/download/v${FORTIO_VERSION}/fortio-linux_amd64-${FORTIO_VERSION}.tgz | |
| tar -xzf fortio-linux_amd64-${FORTIO_VERSION}.tgz | |
| # Store in cache directory | |
| mv usr/bin/fortio ~/bin/ | |
| - name: Cache Vegeta binary | |
| id: cache-vegeta | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/bin/vegeta | |
| key: vegeta-${{ runner.os }}-${{ runner.arch }}-${{ env.VEGETA_VERSION }} | |
| - name: Install Vegeta | |
| if: steps.cache-vegeta.outputs.cache-hit != 'true' | |
| run: | | |
| echo "📦 Installing Vegeta v${VEGETA_VERSION}" | |
| # Download and extract vegeta binary | |
| wget -q https://github.com/tsenart/vegeta/releases/download/v${VEGETA_VERSION}/vegeta_${VEGETA_VERSION}_linux_amd64.tar.gz | |
| tar -xzf vegeta_${VEGETA_VERSION}_linux_amd64.tar.gz | |
| # Store in cache directory | |
| mv vegeta ~/bin/ | |
| - name: Setup k6 | |
| uses: grafana/setup-k6-action@v1 | |
| with: | |
| k6-version: ${{ env.K6_VERSION }} | |
| # ============================================ | |
| # STEP 4: START APPLICATION SERVER | |
| # ============================================ | |
| - name: Setup Ruby | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: '3.4' | |
| bundler: 2.5.9 | |
| - name: Fix dependency for libyaml-dev | |
| run: sudo apt install libyaml-dev -y | |
| - name: Setup Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '22' | |
| cache: yarn | |
| cache-dependency-path: '**/yarn.lock' | |
| - name: Print system information | |
| run: | | |
| echo "Linux release: "; cat /etc/issue | |
| echo "Current user: "; whoami | |
| echo "Current directory: "; pwd | |
| echo "Ruby version: "; ruby -v | |
| echo "Node version: "; node -v | |
| echo "Yarn version: "; yarn --version | |
| echo "Bundler version: "; bundle --version | |
| - name: Install Node modules with Yarn for renderer package | |
| run: | | |
| yarn install --no-progress --no-emoji --frozen-lockfile | |
| npm install --global yalc | |
| - name: yalc publish for react-on-rails | |
| run: cd packages/react-on-rails && yarn install --no-progress --no-emoji --frozen-lockfile && yalc publish | |
| - name: yalc add react-on-rails | |
| run: cd spec/dummy && yalc add react-on-rails | |
| - name: Install Node modules with Yarn for dummy app | |
| run: cd spec/dummy && yarn install --no-progress --no-emoji | |
| - name: Save dummy app ruby gems to cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: spec/dummy/vendor/bundle | |
| key: dummy-app-gem-cache-${{ hashFiles('spec/dummy/Gemfile.lock') }} | |
| - name: Install Ruby Gems for dummy app | |
| run: | | |
| cd spec/dummy | |
| bundle lock --add-platform 'x86_64-linux' | |
| if ! bundle check --path=vendor/bundle; then | |
| bundle _2.5.9_ install --path=vendor/bundle --jobs=4 --retry=3 | |
| fi | |
| - name: generate file system-based packs | |
| run: cd spec/dummy && RAILS_ENV="production" bundle exec rake react_on_rails:generate_packs | |
| - name: Prepare production assets | |
| run: | | |
| set -e # Exit on any error | |
| echo "🔨 Building production assets..." | |
| cd spec/dummy | |
| if ! bin/prod-assets; then | |
| echo "❌ ERROR: Failed to build production assets" | |
| exit 1 | |
| fi | |
| echo "✅ Production assets built successfully" | |
| - name: Start production server | |
| run: | | |
| set -e # Exit on any error | |
| echo "🚀 Starting production server..." | |
| cd spec/dummy | |
| # Start server in background | |
| bin/prod & | |
| echo "Server started in background" | |
| # Wait for server to be ready (max 30 seconds) | |
| echo "⏳ Waiting for server to be ready..." | |
| for i in {1..30}; do | |
| if curl -fsS http://localhost:3001 > /dev/null; then | |
| echo "✅ Server is ready and responding" | |
| exit 0 | |
| fi | |
| echo " Attempt $i/30: Server not ready yet..." | |
| sleep 1 | |
| done | |
| echo "❌ ERROR: Server failed to start within 30 seconds" | |
| exit 1 | |
| # ============================================ | |
| # STEP 5: RUN BENCHMARKS | |
| # ============================================ | |
| - name: Execute benchmark suite | |
| timeout-minutes: 20 | |
| run: | | |
| set -e # Exit on any error | |
| echo "🏃 Running benchmark suite..." | |
| if ! ruby spec/performance/bench.rb; then | |
| echo "❌ ERROR: Benchmark execution failed" | |
| exit 1 | |
| fi | |
| echo "✅ Benchmark suite completed successfully" | |
| - name: Validate benchmark results | |
| run: | | |
| set -e # Exit on any error | |
| echo "🔍 Validating benchmark output files..." | |
| RESULTS_DIR="bench_results" | |
| REQUIRED_FILES=("summary.txt") | |
| MISSING_FILES=() | |
| # Check if results directory exists | |
| if [ ! -d "${RESULTS_DIR}" ]; then | |
| echo "❌ ERROR: Benchmark results directory '${RESULTS_DIR}' not found" | |
| exit 1 | |
| fi | |
| # List all generated files | |
| echo "Generated files:" | |
| ls -lh ${RESULTS_DIR}/ || true | |
| echo "" | |
| # Check for required files | |
| for file in "${REQUIRED_FILES[@]}"; do | |
| if [ ! -f "${RESULTS_DIR}/${file}" ]; then | |
| MISSING_FILES+=("${file}") | |
| fi | |
| done | |
| # Report validation results | |
| if [ ${#MISSING_FILES[@]} -eq 0 ]; then | |
| echo "✅ All required benchmark output files present" | |
| echo "📊 Summary preview:" | |
| head -20 ${RESULTS_DIR}/summary.txt || true | |
| else | |
| echo "⚠️ WARNING: Some required files are missing:" | |
| printf ' - %s\n' "${MISSING_FILES[@]}" | |
| echo "Continuing with available results..." | |
| fi | |
| # ============================================ | |
| # STEP 6: COLLECT BENCHMARK RESULTS | |
| # ============================================ | |
| - name: Upload benchmark results | |
| uses: actions/upload-artifact@v4 | |
| if: always() # Upload even if benchmark fails | |
| with: | |
| name: benchmark-results-${{ github.run_number }} | |
| path: bench_results/ | |
| retention-days: 30 | |
| if-no-files-found: warn | |
| - name: Verify artifact upload | |
| if: success() | |
| run: | | |
| echo "✅ Benchmark results uploaded as workflow artifacts" | |
| echo "📦 Artifact name: benchmark-results-${{ github.run_number }}" | |
| echo "🔗 Access artifacts from the Actions tab in GitHub" | |
| # ============================================ | |
| # WORKFLOW COMPLETION | |
| # ============================================ | |
| - name: Workflow summary | |
| if: always() | |
| run: | | |
| echo "📋 Benchmark Workflow Summary" | |
| echo "==============================" | |
| echo "Status: ${{ job.status }}" | |
| echo "Run number: ${{ github.run_number }}" | |
| echo "Triggered by: ${{ github.actor }}" | |
| echo "Branch: ${{ github.ref_name }}" | |
| echo "" | |
| if [ "${{ job.status }}" == "success" ]; then | |
| echo "✅ All steps completed successfully" | |
| else | |
| echo "❌ Workflow encountered errors - check logs above" | |
| fi |