Add performance tests #44
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark Workflow | |
| on: | |
| # https://github.com/mxschmitt/action-tmate?tab=readme-ov-file#manually-triggered-debug | |
| workflow_dispatch: | |
| inputs: | |
| debug_enabled: | |
| description: 'Enable SSH access (⚠️ Security Risk - read workflow comments)' | |
| required: false | |
| default: false | |
| type: boolean | |
| routes: | |
| description: 'Comma-separated routes to benchmark (e.g., "/,/hello"). Leave empty to auto-detect from Rails.' | |
| required: false | |
| type: string | |
| rate: | |
| description: 'Requests per second (use "max" for maximum throughput)' | |
| required: false | |
| default: 'max' | |
| type: string | |
| duration: | |
| description: 'Duration (e.g., "30s", "1m", "90s")' | |
| required: false | |
| default: '30s' | |
| type: string | |
| request_timeout: | |
| description: 'Request timeout (e.g., "60s", "1m", "90s")' | |
| required: false | |
| default: '60s' | |
| type: string | |
| connections: | |
| description: 'Concurrent connections/virtual users (also used as max)' | |
| required: false | |
| default: 10 | |
| type: number | |
| web_concurrency: | |
| description: 'Number of Puma worker processes' | |
| required: false | |
| default: 4 | |
| type: number | |
| rails_threads: | |
| description: 'Number of Puma threads (min and max will be same)' | |
| required: false | |
| default: 3 | |
| type: number | |
| tools: | |
| description: 'Comma-separated list of tools to run' | |
| required: false | |
| default: 'fortio,vegeta,k6' | |
| type: string | |
| app_version: | |
| description: 'Which app version to benchmark' | |
| required: false | |
| # FIXME: for debugging, restore 'both' before merging | |
| default: 'pro_only' | |
| type: choice | |
| options: | |
| - 'both' | |
| - 'core_only' | |
| - 'pro_only' | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| env: | |
| FORTIO_VERSION: "1.73.0" | |
| K6_VERSION: "1.3.0" | |
| VEGETA_VERSION: "12.13.0" | |
| # Benchmark parameters | |
| ROUTES: ${{ github.event.inputs.routes }} | |
| RATE: ${{ github.event.inputs.rate || 'max' }} | |
| DURATION: ${{ github.event.inputs.duration || '30s' }} | |
| REQUEST_TIMEOUT: ${{ github.event.inputs.request_timeout || '60s' }} | |
| CONNECTIONS: ${{ github.event.inputs.connections || 10 }} | |
| MAX_CONNECTIONS: ${{ github.event.inputs.connections || 10 }} | |
| WEB_CONCURRENCY: ${{ github.event.inputs.web_concurrency || 4 }} | |
| RAILS_MAX_THREADS: ${{ github.event.inputs.rails_threads || 3 }} | |
| RAILS_MIN_THREADS: ${{ github.event.inputs.rails_threads || 3 }} | |
| TOOLS: ${{ github.event.inputs.tools || 'fortio,vegeta,k6' }} | |
| jobs: | |
| benchmark: | |
| runs-on: ubuntu-latest | |
| env: | |
| SECRET_KEY_BASE: 'dummy-secret-key-for-ci-testing-not-used-in-production' | |
| REACT_ON_RAILS_PRO_LICENSE: ${{ secrets.REACT_ON_RAILS_PRO_LICENSE_V2 }} | |
| steps: | |
| # ============================================ | |
| # STEP 1: CHECKOUT CODE | |
| # ============================================ | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| # ============================================ | |
| # STEP 2: OPTIONAL SSH ACCESS | |
| # ============================================ | |
| # NOTE: Interactive confirmation is not possible in GitHub Actions. | |
| # As a secure workaround, SSH access is gated by the workflow_dispatch | |
| # input variable 'debug_enabled' which defaults to false. | |
| # Users must explicitly set this to true to enable SSH. | |
| - name: SSH Warning | |
| if: ${{ github.event.inputs.debug_enabled == true || github.event.inputs.debug_enabled == 'true' }} | |
| run: | | |
| echo "⚠️ ⚠️ ⚠️ SSH ACCESS ENABLED ⚠️ ⚠️ ⚠️" | |
| echo "" | |
| echo "SECURITY NOTICE:" | |
| echo " - SSH access exposes your GitHub Actions runner" | |
| echo " - Only proceed if you understand and accept the risks" | |
| echo " - Do NOT store secrets or sensitive data on the runner" | |
| echo " - Access is limited to the workflow initiator only" | |
| echo " - The session will remain open until manually terminated" | |
| echo "" | |
| echo "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️" | |
| - name: Setup SSH access (if enabled) | |
| if: ${{ github.event.inputs.debug_enabled == true || github.event.inputs.debug_enabled == 'true' }} | |
| uses: mxschmitt/action-tmate@v3 | |
| with: | |
| detached: true | |
| limit-access-to-actor: true # Only workflow trigger can access | |
| # ============================================ | |
| # STEP 3: INSTALL BENCHMARKING TOOLS | |
| # ============================================ | |
| - name: Add tools directory to PATH | |
| run: | | |
| mkdir -p ~/bin | |
| echo "$HOME/bin" >> $GITHUB_PATH | |
| - name: Cache Fortio binary | |
| id: cache-fortio | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/bin/fortio | |
| key: fortio-${{ runner.os }}-${{ runner.arch }}-${{ env.FORTIO_VERSION }} | |
| - name: Install Fortio | |
| if: steps.cache-fortio.outputs.cache-hit != 'true' | |
| run: | | |
| echo "📦 Installing Fortio v${FORTIO_VERSION}" | |
| # Download and extract fortio binary | |
| wget -q https://github.com/fortio/fortio/releases/download/v${FORTIO_VERSION}/fortio-linux_amd64-${FORTIO_VERSION}.tgz | |
| tar -xzf fortio-linux_amd64-${FORTIO_VERSION}.tgz | |
| # Store in cache directory | |
| mv usr/bin/fortio ~/bin/ | |
| - name: Cache Vegeta binary | |
| id: cache-vegeta | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/bin/vegeta | |
| key: vegeta-${{ runner.os }}-${{ runner.arch }}-${{ env.VEGETA_VERSION }} | |
| - name: Install Vegeta | |
| if: steps.cache-vegeta.outputs.cache-hit != 'true' | |
| run: | | |
| echo "📦 Installing Vegeta v${VEGETA_VERSION}" | |
| # Download and extract vegeta binary | |
| wget -q https://github.com/tsenart/vegeta/releases/download/v${VEGETA_VERSION}/vegeta_${VEGETA_VERSION}_linux_amd64.tar.gz | |
| tar -xzf vegeta_${VEGETA_VERSION}_linux_amd64.tar.gz | |
| # Store in cache directory | |
| mv vegeta ~/bin/ | |
| - name: Setup k6 | |
| uses: grafana/setup-k6-action@v1 | |
| with: | |
| k6-version: ${{ env.K6_VERSION }} | |
| # ============================================ | |
| # STEP 4: START APPLICATION SERVER | |
| # ============================================ | |
| - name: Setup Ruby | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: '3.3.7' | |
| bundler: 2.5.4 | |
| - name: Fix dependency for libyaml-dev | |
| run: sudo apt install libyaml-dev -y | |
| - name: Setup Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '22' | |
| cache: yarn | |
| cache-dependency-path: '**/yarn.lock' | |
| - name: Print system information | |
| run: | | |
| echo "Linux release: "; cat /etc/issue | |
| echo "Current user: "; whoami | |
| echo "Current directory: "; pwd | |
| echo "Ruby version: "; ruby -v | |
| echo "Node version: "; node -v | |
| echo "Yarn version: "; yarn --version | |
| echo "Bundler version: "; bundle --version | |
| - name: Install Node modules with Yarn for renderer package | |
| run: | | |
| yarn install --no-progress --no-emoji --frozen-lockfile | |
| npm install --global yalc | |
| - name: yalc publish for react-on-rails | |
| run: cd packages/react-on-rails && yarn install --no-progress --no-emoji --frozen-lockfile && yalc publish | |
| - name: yalc add react-on-rails | |
| if: github.event.inputs.app_version != 'pro_only' | |
| run: cd spec/dummy && yalc add react-on-rails | |
| - name: Install Node modules with Yarn for dummy app | |
| if: github.event.inputs.app_version != 'pro_only' | |
| run: cd spec/dummy && yarn install --no-progress --no-emoji | |
| - name: Save dummy app ruby gems to cache | |
| if: github.event.inputs.app_version != 'pro_only' | |
| uses: actions/cache@v4 | |
| with: | |
| path: spec/dummy/vendor/bundle | |
| key: dummy-app-gem-cache-${{ hashFiles('spec/dummy/Gemfile.lock') }} | |
| - name: Install Ruby Gems for dummy app | |
| if: github.event.inputs.app_version != 'pro_only' | |
| run: | | |
| cd spec/dummy | |
| bundle lock --add-platform 'x86_64-linux' | |
| if ! bundle check --path=vendor/bundle; then | |
| bundle _2.5.4_ install --path=vendor/bundle --jobs=4 --retry=3 | |
| fi | |
| - name: Prepare production assets | |
| if: github.event.inputs.app_version != 'pro_only' | |
| run: | | |
| set -e # Exit on any error | |
| echo "🔨 Building production assets..." | |
| cd spec/dummy | |
| if ! bin/prod-assets; then | |
| echo "❌ ERROR: Failed to build production assets" | |
| exit 1 | |
| fi | |
| echo "✅ Production assets built successfully" | |
| - name: Start production server | |
| if: github.event.inputs.app_version != 'pro_only' | |
| run: | | |
| set -e # Exit on any error | |
| echo "🚀 Starting production server..." | |
| cd spec/dummy | |
| # Start server in background | |
| bin/prod & | |
| echo "Server started in background" | |
| # Wait for server to be ready (max 30 seconds) | |
| echo "⏳ Waiting for server to be ready..." | |
| for i in {1..30}; do | |
| if curl -fsS http://localhost:3001 > /dev/null; then | |
| echo "✅ Server is ready and responding" | |
| exit 0 | |
| fi | |
| echo " Attempt $i/30: Server not ready yet..." | |
| sleep 1 | |
| done | |
| echo "❌ ERROR: Server failed to start within 30 seconds" | |
| exit 1 | |
| # ============================================ | |
| # STEP 5: RUN CORE BENCHMARKS | |
| # ============================================ | |
| - name: Execute Core benchmark suite | |
| if: github.event.inputs.app_version != 'pro_only' | |
| timeout-minutes: 120 | |
| run: | | |
| set -e # Exit on any error | |
| echo "🏃 Running Core benchmark suite..." | |
| if ! ruby spec/performance/bench.rb; then | |
| echo "❌ ERROR: Benchmark execution failed" | |
| exit 1 | |
| fi | |
| echo "✅ Benchmark suite completed successfully" | |
| - name: Validate Core benchmark results | |
| if: github.event.inputs.app_version != 'pro_only' | |
| run: | | |
| set -e # Exit on any error | |
| echo "🔍 Validating benchmark output files..." | |
| RESULTS_DIR="bench_results" | |
| REQUIRED_FILES=("summary.txt") | |
| MISSING_FILES=() | |
| # Check if results directory exists | |
| if [ ! -d "${RESULTS_DIR}" ]; then | |
| echo "❌ ERROR: Benchmark results directory '${RESULTS_DIR}' not found" | |
| exit 1 | |
| fi | |
| # List all generated files | |
| echo "Generated files:" | |
| ls -lh ${RESULTS_DIR}/ || true | |
| echo "" | |
| # Check for required files | |
| for file in "${REQUIRED_FILES[@]}"; do | |
| if [ ! -f "${RESULTS_DIR}/${file}" ]; then | |
| MISSING_FILES+=("${file}") | |
| fi | |
| done | |
| # Report validation results | |
| if [ ${#MISSING_FILES[@]} -eq 0 ]; then | |
| echo "✅ All required benchmark output files present" | |
| echo "📊 Summary preview:" | |
| head -20 ${RESULTS_DIR}/summary.txt || true | |
| else | |
| echo "⚠️ WARNING: Some required files are missing:" | |
| printf ' - %s\n' "${MISSING_FILES[@]}" | |
| echo "Continuing with available results..." | |
| fi | |
| - name: Upload Core benchmark results | |
| uses: actions/upload-artifact@v4 | |
| if: github.event.inputs.app_version != 'pro_only' && always() | |
| with: | |
| name: benchmark-core-results-${{ github.run_number }} | |
| path: bench_results/ | |
| retention-days: 30 | |
| if-no-files-found: warn | |
| # ============================================ | |
| # STEP 6: SETUP PRO APPLICATION SERVER | |
| # ============================================ | |
| - name: Cache Pro package node modules | |
| if: github.event.inputs.app_version != 'core_only' | |
| uses: actions/cache@v4 | |
| with: | |
| path: react_on_rails_pro/node_modules | |
| key: v4-pro-package-node-modules-cache-${{ hashFiles('react_on_rails_pro/yarn.lock') }} | |
| - name: Cache Pro dummy app node modules | |
| if: github.event.inputs.app_version != 'core_only' | |
| uses: actions/cache@v4 | |
| with: | |
| path: react_on_rails_pro/spec/dummy/node_modules | |
| key: v4-pro-dummy-app-node-modules-cache-${{ hashFiles('react_on_rails_pro/spec/dummy/yarn.lock') }} | |
| - name: Cache Pro dummy app Ruby gems | |
| if: github.event.inputs.app_version != 'core_only' | |
| uses: actions/cache@v4 | |
| with: | |
| path: react_on_rails_pro/spec/dummy/vendor/bundle | |
| key: v4-pro-dummy-app-gem-cache-${{ hashFiles('react_on_rails_pro/spec/dummy/Gemfile.lock') }} | |
| - name: Install Node modules with Yarn for Pro package | |
| if: github.event.inputs.app_version != 'core_only' | |
| run: | | |
| cd react_on_rails_pro | |
| sudo yarn global add yalc | |
| yarn install --frozen-lockfile --no-progress --no-emoji | |
| - name: Install Node modules with Yarn for Pro dummy app | |
| if: github.event.inputs.app_version != 'core_only' | |
| run: cd react_on_rails_pro/spec/dummy && yarn install --frozen-lockfile --no-progress --no-emoji | |
| - name: Install Ruby Gems for Pro dummy app | |
| if: github.event.inputs.app_version != 'core_only' | |
| run: | | |
| cd react_on_rails_pro/spec/dummy | |
| bundle lock --add-platform 'x86_64-linux' | |
| bundle _2.5.4_ check || bundle _2.5.4_ install --jobs=4 --retry=3 | |
| - name: Generate file-system based entrypoints for Pro | |
| if: github.event.inputs.app_version != 'core_only' | |
| run: cd react_on_rails_pro/spec/dummy && bundle exec rake react_on_rails:generate_packs | |
| - name: Prepare Pro production assets | |
| if: github.event.inputs.app_version != 'core_only' | |
| run: | | |
| set -e | |
| echo "🔨 Building Pro production assets..." | |
| cd react_on_rails_pro/spec/dummy | |
| if ! bin/prod-assets; then | |
| echo "❌ ERROR: Failed to build production assets" | |
| exit 1 | |
| fi | |
| echo "✅ Production assets built successfully" | |
| - name: Start Pro production server | |
| if: github.event.inputs.app_version != 'core_only' | |
| run: | | |
| set -e | |
| echo "🚀 Starting Pro production server..." | |
| cd react_on_rails_pro/spec/dummy | |
| # Start server in background | |
| bin/prod & | |
| echo "Server started in background" | |
| # Wait for server to be ready (max 30 seconds) | |
| echo "⏳ Waiting for server to be ready..." | |
| for i in {1..30}; do | |
| if curl -fsS http://localhost:3001 > /dev/null; then | |
| echo "✅ Server is ready and responding" | |
| exit 0 | |
| fi | |
| echo " Attempt $i/30: Server not ready yet..." | |
| sleep 1 | |
| done | |
| echo "❌ ERROR: Server failed to start within 30 seconds" | |
| exit 1 | |
| # ============================================ | |
| # STEP 7: RUN PRO BENCHMARKS | |
| # ============================================ | |
| - name: Execute Pro benchmark suite | |
| if: github.event.inputs.app_version != 'core_only' | |
| timeout-minutes: 120 | |
| run: | | |
| set -e | |
| echo "🏃 Running Pro benchmark suite..." | |
| if ! PRO=true ruby spec/performance/bench.rb; then | |
| echo "❌ ERROR: Benchmark execution failed" | |
| exit 1 | |
| fi | |
| echo "✅ Benchmark suite completed successfully" | |
| - name: Validate Pro benchmark results | |
| if: github.event.inputs.app_version != 'core_only' | |
| run: | | |
| set -e | |
| echo "🔍 Validating Pro benchmark output files..." | |
| RESULTS_DIR="bench_results" | |
| REQUIRED_FILES=("summary.txt") | |
| MISSING_FILES=() | |
| if [ ! -d "${RESULTS_DIR}" ]; then | |
| echo "❌ ERROR: Benchmark results directory '${RESULTS_DIR}' not found" | |
| exit 1 | |
| fi | |
| echo "Generated files:" | |
| ls -lh ${RESULTS_DIR}/ || true | |
| echo "" | |
| for file in "${REQUIRED_FILES[@]}"; do | |
| if [ ! -f "${RESULTS_DIR}/${file}" ]; then | |
| MISSING_FILES+=("${file}") | |
| fi | |
| done | |
| if [ ${#MISSING_FILES[@]} -eq 0 ]; then | |
| echo "✅ All required benchmark output files present" | |
| echo "📊 Summary preview:" | |
| head -20 ${RESULTS_DIR}/summary.txt || true | |
| else | |
| echo "⚠️ WARNING: Some required files are missing:" | |
| printf ' - %s\n' "${MISSING_FILES[@]}" | |
| echo "Continuing with available results..." | |
| fi | |
| - name: Upload Pro benchmark results | |
| uses: actions/upload-artifact@v4 | |
| if: github.event.inputs.app_version != 'core_only' && always() | |
| with: | |
| name: benchmark-pro-results-${{ github.run_number }} | |
| path: bench_results/ | |
| retention-days: 30 | |
| if-no-files-found: warn | |
| # ============================================ | |
| # STEP 8: WORKFLOW COMPLETION | |
| # ============================================ | |
| - name: Workflow summary | |
| if: always() | |
| run: | | |
| echo "📋 Benchmark Workflow Summary" | |
| echo "====================================" | |
| echo "Status: ${{ job.status }}" | |
| echo "Run number: ${{ github.run_number }}" | |
| echo "Triggered by: ${{ github.actor }}" | |
| echo "Branch: ${{ github.ref_name }}" | |
| echo "App version: ${{ github.event.inputs.app_version || 'both' }}" | |
| echo "" | |
| if [ "${{ job.status }}" == "success" ]; then | |
| echo "✅ All steps completed successfully" | |
| else | |
| echo "❌ Workflow encountered errors - check logs above" | |
| fi |