Add performance tests #55
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark Workflow | |
| on: | |
| # https://github.com/mxschmitt/action-tmate?tab=readme-ov-file#manually-triggered-debug | |
| workflow_dispatch: | |
| inputs: | |
| debug_enabled: | |
| description: 'Enable SSH access (⚠️ Security Risk - read workflow comments)' | |
| required: false | |
| default: false | |
| type: boolean | |
| routes: | |
| description: 'Comma-separated routes to benchmark (e.g., "/,/hello"). Leave empty to auto-detect from Rails.' | |
| required: false | |
| type: string | |
| rate: | |
| description: 'Requests per second (use "max" for maximum throughput)' | |
| required: false | |
| default: 'max' | |
| type: string | |
| duration: | |
| description: 'Duration (e.g., "30s", "1m", "90s")' | |
| required: false | |
| default: '30s' | |
| type: string | |
| request_timeout: | |
| description: 'Request timeout (e.g., "60s", "1m", "90s")' | |
| required: false | |
| default: '60s' | |
| type: string | |
| connections: | |
| description: 'Concurrent connections/virtual users (also used as max)' | |
| required: false | |
| default: 10 | |
| type: number | |
| web_concurrency: | |
| description: 'Number of Puma worker processes' | |
| required: false | |
| default: 4 | |
| type: number | |
| rails_threads: | |
| description: 'Number of Puma threads (min and max will be same)' | |
| required: false | |
| default: 3 | |
| type: number | |
| tools: | |
| description: 'Comma-separated list of tools to run' | |
| required: false | |
| default: 'fortio,vegeta,k6' | |
| type: string | |
| app_version: | |
| description: 'Which app version to benchmark' | |
| required: false | |
| default: 'both' | |
| type: choice | |
| options: | |
| - 'both' | |
| - 'core_only' | |
| - 'pro_only' | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| env: | |
| FORTIO_VERSION: "1.73.0" | |
| K6_VERSION: "1.3.0" | |
| VEGETA_VERSION: "12.13.0" | |
| # Determine which apps to run (default is 'pro_only' for all triggers) | |
| RUN_CORE: ${{ (github.event.inputs.app_version || 'both') != 'pro_only' && 'true' || '' }} | |
| RUN_PRO: ${{ (github.event.inputs.app_version || 'both') != 'core_only' && 'true' || '' }} | |
| # Benchmark parameters (defaults in bench.rb unless overridden here for CI) | |
| # FIXME: default ROUTES, TOOLS and DURATION are set to speed up tests, remove before merging | |
| ROUTES: ${{ github.event.inputs.routes || '/' }} | |
| RATE: ${{ github.event.inputs.rate || 'max' }} | |
| DURATION: ${{ github.event.inputs.duration || '5s' }} | |
| REQUEST_TIMEOUT: ${{ github.event.inputs.request_timeout }} | |
| CONNECTIONS: ${{ github.event.inputs.connections }} | |
| MAX_CONNECTIONS: ${{ github.event.inputs.connections }} | |
| WEB_CONCURRENCY: ${{ github.event.inputs.web_concurrency || 4 }} | |
| RAILS_MAX_THREADS: ${{ github.event.inputs.rails_threads || 3 }} | |
| RAILS_MIN_THREADS: ${{ github.event.inputs.rails_threads || 3 }} | |
| TOOLS: ${{ github.event.inputs.tools || 'fortio' }} | |
| jobs: | |
| benchmark: | |
| runs-on: ubuntu-latest | |
| env: | |
| SECRET_KEY_BASE: 'dummy-secret-key-for-ci-testing-not-used-in-production' | |
| REACT_ON_RAILS_PRO_LICENSE: ${{ secrets.REACT_ON_RAILS_PRO_LICENSE_V2 }} | |
| steps: | |
| # ============================================ | |
| # STEP 1: CHECKOUT CODE | |
| # ============================================ | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| # ============================================ | |
| # STEP 2: OPTIONAL SSH ACCESS | |
| # ============================================ | |
| # NOTE: Interactive confirmation is not possible in GitHub Actions. | |
| # As a secure workaround, SSH access is gated by the workflow_dispatch | |
| # input variable 'debug_enabled' which defaults to false. | |
| # Users must explicitly set this to true to enable SSH. | |
| - name: SSH Warning | |
| if: ${{ github.event.inputs.debug_enabled == true || github.event.inputs.debug_enabled == 'true' }} | |
| run: | | |
| echo "⚠️ ⚠️ ⚠️ SSH ACCESS ENABLED ⚠️ ⚠️ ⚠️" | |
| echo "" | |
| echo "SECURITY NOTICE:" | |
| echo " - SSH access exposes your GitHub Actions runner" | |
| echo " - Only proceed if you understand and accept the risks" | |
| echo " - Do NOT store secrets or sensitive data on the runner" | |
| echo " - Access is limited to the workflow initiator only" | |
| echo " - The session will remain open until manually terminated" | |
| echo "" | |
| echo "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️" | |
| - name: Setup SSH access (if enabled) | |
| if: ${{ github.event.inputs.debug_enabled == true || github.event.inputs.debug_enabled == 'true' }} | |
| uses: mxschmitt/action-tmate@v3 | |
| with: | |
| detached: true | |
| limit-access-to-actor: true # Only workflow trigger can access | |
| # ============================================ | |
| # STEP 3: INSTALL BENCHMARKING TOOLS | |
| # ============================================ | |
| - name: Add tools directory to PATH | |
| run: | | |
| mkdir -p ~/bin | |
| echo "$HOME/bin" >> $GITHUB_PATH | |
| - name: Cache Fortio binary | |
| id: cache-fortio | |
| if: contains(env.TOOLS, 'fortio') | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/bin/fortio | |
| key: fortio-${{ runner.os }}-${{ runner.arch }}-${{ env.FORTIO_VERSION }} | |
| - name: Install Fortio | |
| if: contains(env.TOOLS, 'fortio') && steps.cache-fortio.outputs.cache-hit != 'true' | |
| run: | | |
| echo "📦 Installing Fortio v${FORTIO_VERSION}" | |
| # Download and extract fortio binary | |
| wget -q https://github.com/fortio/fortio/releases/download/v${FORTIO_VERSION}/fortio-linux_amd64-${FORTIO_VERSION}.tgz | |
| tar -xzf fortio-linux_amd64-${FORTIO_VERSION}.tgz | |
| # Store in cache directory | |
| mv usr/bin/fortio ~/bin/ | |
| - name: Cache Vegeta binary | |
| id: cache-vegeta | |
| if: contains(env.TOOLS, 'vegeta') | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/bin/vegeta | |
| key: vegeta-${{ runner.os }}-${{ runner.arch }}-${{ env.VEGETA_VERSION }} | |
| - name: Install Vegeta | |
| if: contains(env.TOOLS, 'vegeta') && steps.cache-vegeta.outputs.cache-hit != 'true' | |
| run: | | |
| echo "📦 Installing Vegeta v${VEGETA_VERSION}" | |
| # Download and extract vegeta binary | |
| wget -q https://github.com/tsenart/vegeta/releases/download/v${VEGETA_VERSION}/vegeta_${VEGETA_VERSION}_linux_amd64.tar.gz | |
| tar -xzf vegeta_${VEGETA_VERSION}_linux_amd64.tar.gz | |
| # Store in cache directory | |
| mv vegeta ~/bin/ | |
| - name: Setup k6 | |
| if: contains(env.TOOLS, 'k6') | |
| uses: grafana/setup-k6-action@v1 | |
| with: | |
| k6-version: ${{ env.K6_VERSION }} | |
| # ============================================ | |
| # STEP 4: START APPLICATION SERVER | |
| # ============================================ | |
| - name: Setup Ruby | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: '3.3.7' | |
| bundler: 2.5.4 | |
| - name: Fix dependency for libyaml-dev | |
| run: sudo apt install libyaml-dev -y | |
| - name: Setup Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '22' | |
| cache: yarn | |
| cache-dependency-path: '**/yarn.lock' | |
| - name: Print system information | |
| run: | | |
| echo "Linux release: "; cat /etc/issue | |
| echo "Current user: "; whoami | |
| echo "Current directory: "; pwd | |
| echo "Ruby version: "; ruby -v | |
| echo "Node version: "; node -v | |
| echo "Yarn version: "; yarn --version | |
| echo "Bundler version: "; bundle --version | |
| - name: Install Node modules with Yarn for renderer package | |
| run: | | |
| yarn install --no-progress --no-emoji --frozen-lockfile | |
| npm install --global yalc | |
| - name: yalc publish for react-on-rails | |
| run: cd packages/react-on-rails && yarn install --no-progress --no-emoji --frozen-lockfile && yalc publish | |
| - name: yalc add react-on-rails | |
| if: env.RUN_CORE | |
| run: cd spec/dummy && yalc add react-on-rails | |
| - name: Install Node modules with Yarn for Core dummy app | |
| if: env.RUN_CORE | |
| run: cd spec/dummy && yarn install --no-progress --no-emoji | |
| - name: Save Core dummy app ruby gems to cache | |
| if: env.RUN_CORE | |
| uses: actions/cache@v4 | |
| with: | |
| path: spec/dummy/vendor/bundle | |
| key: dummy-app-gem-cache-${{ hashFiles('spec/dummy/Gemfile.lock') }} | |
| - name: Install Ruby Gems for Core dummy app | |
| if: env.RUN_CORE | |
| run: | | |
| cd spec/dummy | |
| bundle lock --add-platform 'x86_64-linux' | |
| if ! bundle check --path=vendor/bundle; then | |
| bundle _2.5.4_ install --path=vendor/bundle --jobs=4 --retry=3 | |
| fi | |
| - name: Prepare Core production assets | |
| if: env.RUN_CORE | |
| run: | | |
| set -e # Exit on any error | |
| echo "🔨 Building production assets..." | |
| cd spec/dummy | |
| if ! bin/prod-assets; then | |
| echo "❌ ERROR: Failed to build production assets" | |
| exit 1 | |
| fi | |
| echo "✅ Production assets built successfully" | |
| - name: Start Core production server | |
| if: env.RUN_CORE | |
| run: | | |
| set -e # Exit on any error | |
| echo "🚀 Starting production server..." | |
| cd spec/dummy | |
| # Start server in background with nohup to persist after step ends | |
| nohup bin/prod > server.log 2>&1 & | |
| SERVER_PID=$! | |
| disown | |
| echo "Server started in background (PID: $SERVER_PID)" | |
| # Wait for server to be ready (max 30 seconds) | |
| echo "⏳ Waiting for server to be ready..." | |
| for i in {1..30}; do | |
| if curl -fsS http://localhost:3001 > /dev/null; then | |
| echo "✅ Server is ready and responding" | |
| break | |
| fi | |
| echo " Attempt $i/30: Server not ready yet..." | |
| sleep 1 | |
| done | |
| # Check if server is actually responding | |
| if ! curl -fsS http://localhost:3001 > /dev/null; then | |
| echo "❌ ERROR: Server failed to start within 30 seconds" | |
| exit 1 | |
| fi | |
| # ============================================ | |
| # STEP 5: RUN CORE BENCHMARKS | |
| # ============================================ | |
| - name: Execute Core benchmark suite | |
| if: env.RUN_CORE | |
| timeout-minutes: 120 | |
| run: | | |
| set -e # Exit on any error | |
| echo "🏃 Running Core benchmark suite..." | |
| if ! ruby spec/performance/bench.rb; then | |
| echo "❌ ERROR: Benchmark execution failed" | |
| exit 1 | |
| fi | |
| echo "✅ Benchmark suite completed successfully" | |
| - name: Stop Core production server | |
| if: env.RUN_CORE && always() | |
| run: | | |
| echo "🛑 Stopping Core production server..." | |
| # Find and kill the Puma process on port 3001 | |
| if lsof -ti:3001 > /dev/null 2>&1; then | |
| kill $(lsof -ti:3001) || true | |
| echo "✅ Server stopped" | |
| else | |
| echo "ℹ️ No server running on port 3001" | |
| fi | |
| - name: Validate Core benchmark results | |
| if: env.RUN_CORE | |
| run: | | |
| set -e | |
| echo "🔍 Validating benchmark results..." | |
| if [ ! -f "bench_results/summary.txt" ]; then | |
| echo "❌ ERROR: benchmark summary file not found" | |
| exit 1 | |
| fi | |
| echo "✅ Benchmark results found" | |
| echo "" | |
| echo "📊 Summary:" | |
| column -t -s $'\t' bench_results/summary.txt | |
| echo "" | |
| echo "Generated files:" | |
| ls -lh bench_results/ | |
| - name: Upload Core benchmark results | |
| uses: actions/upload-artifact@v4 | |
| if: env.RUN_CORE && always() | |
| with: | |
| name: benchmark-core-results-${{ github.run_number }} | |
| path: bench_results/ | |
| retention-days: 30 | |
| if-no-files-found: warn | |
| # ============================================ | |
| # STEP 6: SETUP PRO APPLICATION SERVER | |
| # ============================================ | |
| - name: Cache Pro package node modules | |
| if: env.RUN_PRO | |
| uses: actions/cache@v4 | |
| with: | |
| path: react_on_rails_pro/node_modules | |
| key: v4-pro-package-node-modules-cache-${{ hashFiles('react_on_rails_pro/yarn.lock') }} | |
| - name: Cache Pro dummy app node modules | |
| if: env.RUN_PRO | |
| uses: actions/cache@v4 | |
| with: | |
| path: react_on_rails_pro/spec/dummy/node_modules | |
| key: v4-pro-dummy-app-node-modules-cache-${{ hashFiles('react_on_rails_pro/spec/dummy/yarn.lock') }} | |
| - name: Cache Pro dummy app Ruby gems | |
| if: env.RUN_PRO | |
| uses: actions/cache@v4 | |
| with: | |
| path: react_on_rails_pro/spec/dummy/vendor/bundle | |
| key: v4-pro-dummy-app-gem-cache-${{ hashFiles('react_on_rails_pro/spec/dummy/Gemfile.lock') }} | |
| - name: Install Node modules with Yarn for Pro package | |
| if: env.RUN_PRO | |
| run: | | |
| cd react_on_rails_pro | |
| sudo yarn global add yalc | |
| yarn install --frozen-lockfile --no-progress --no-emoji | |
| - name: Install Node modules with Yarn for Pro dummy app | |
| if: env.RUN_PRO | |
| run: cd react_on_rails_pro/spec/dummy && yarn install --frozen-lockfile --no-progress --no-emoji | |
| - name: Install Ruby Gems for Pro dummy app | |
| if: env.RUN_PRO | |
| run: | | |
| cd react_on_rails_pro/spec/dummy | |
| bundle lock --add-platform 'x86_64-linux' | |
| bundle _2.5.4_ check || bundle _2.5.4_ install --jobs=4 --retry=3 | |
| - name: Generate file-system based entrypoints for Pro | |
| if: env.RUN_PRO | |
| run: cd react_on_rails_pro/spec/dummy && bundle exec rake react_on_rails:generate_packs | |
| - name: Prepare Pro production assets | |
| if: env.RUN_PRO | |
| run: | | |
| set -e | |
| echo "🔨 Building Pro production assets..." | |
| cd react_on_rails_pro/spec/dummy | |
| if ! bin/prod-assets; then | |
| echo "❌ ERROR: Failed to build production assets" | |
| exit 1 | |
| fi | |
| echo "✅ Production assets built successfully" | |
| - name: Start Pro production server | |
| if: env.RUN_PRO | |
| run: | | |
| set -e | |
| echo "🚀 Starting Pro production server..." | |
| cd react_on_rails_pro/spec/dummy | |
| # Start server in background with nohup to persist after step ends | |
| nohup bin/prod > server.log 2>&1 & | |
| SERVER_PID=$! | |
| disown | |
| echo "Server started in background (PID: $SERVER_PID)" | |
| # Wait for server to be ready (max 30 seconds) | |
| echo "⏳ Waiting for server to be ready..." | |
| for i in {1..30}; do | |
| if curl -fsS http://localhost:3001 > /dev/null; then | |
| echo "✅ Server is ready and responding" | |
| break | |
| fi | |
| echo " Attempt $i/30: Server not ready yet..." | |
| sleep 1 | |
| done | |
| # Check if server is actually responding | |
| if ! curl -fsS http://localhost:3001 > /dev/null; then | |
| echo "❌ ERROR: Server failed to start within 30 seconds" | |
| exit 1 | |
| fi | |
| # ============================================ | |
| # STEP 7: RUN PRO BENCHMARKS | |
| # ============================================ | |
| - name: Execute Pro benchmark suite | |
| if: env.RUN_PRO | |
| timeout-minutes: 120 | |
| run: | | |
| set -e | |
| echo "🏃 Running Pro benchmark suite..." | |
| if ! PRO=true ruby spec/performance/bench.rb; then | |
| echo "❌ ERROR: Benchmark execution failed" | |
| exit 1 | |
| fi | |
| echo "✅ Benchmark suite completed successfully" | |
| - name: Stop Pro production server | |
| if: env.RUN_PRO && always() | |
| run: | | |
| echo "🛑 Stopping Pro production server..." | |
| # Find and kill the Puma process on port 3001 | |
| if lsof -ti:3001 > /dev/null 2>&1; then | |
| kill $(lsof -ti:3001) || true | |
| echo "✅ Server stopped" | |
| else | |
| echo "ℹ️ No server running on port 3001" | |
| fi | |
| - name: Validate Pro benchmark results | |
| if: env.RUN_PRO | |
| run: | | |
| set -e | |
| echo "🔍 Validating benchmark results..." | |
| if [ ! -f "bench_results/summary.txt" ]; then | |
| echo "❌ ERROR: benchmark summary file not found" | |
| exit 1 | |
| fi | |
| echo "✅ Benchmark results found" | |
| echo "" | |
| echo "📊 Summary:" | |
| column -t -s $'\t' bench_results/summary.txt | |
| echo "" | |
| echo "Generated files:" | |
| ls -lh bench_results/ | |
| - name: Upload Pro benchmark results | |
| uses: actions/upload-artifact@v4 | |
| if: env.RUN_PRO && always() | |
| with: | |
| name: benchmark-pro-results-${{ github.run_number }} | |
| path: bench_results/ | |
| retention-days: 30 | |
| if-no-files-found: warn | |
| # ============================================ | |
| # STEP 8: WORKFLOW COMPLETION | |
| # ============================================ | |
| - name: Workflow summary | |
| if: always() | |
| run: | | |
| echo "📋 Benchmark Workflow Summary" | |
| echo "====================================" | |
| echo "Status: ${{ job.status }}" | |
| echo "Run number: ${{ github.run_number }}" | |
| echo "Triggered by: ${{ github.actor }}" | |
| echo "Branch: ${{ github.ref_name }}" | |
| echo "Run Core: ${{ env.RUN_CORE }}" | |
| echo "Run Pro: ${{ env.RUN_PRO }}" | |
| echo "" | |
| if [ "${{ job.status }}" == "success" ]; then | |
| echo "✅ All steps completed successfully" | |
| else | |
| echo "❌ Workflow encountered errors - check logs above" | |
| fi |