Setup Continuous Performance

A new workflow, continuous-integration-perf.yml was introduced. It:

 - Checks out the `benchmarks` branch locally.
 - Runs the benchmarks, accounting for non-existant baselines
   and target (main/PR).
 - Stores the .phpbench storage folder and a human-readable
   report in the `benchmarks` branch.
 - Does not make a PR fail, and never reports a failure
   when merging to main.
 - Allows workflow_dispatch for quick re-runs, and has an
   option to reset the baseline in case something changes
   (GitHub runner setup gets faster/slower, major refactors,
   etc).

Thus, it keeps a historical record of all benchmark results.

These results can be viewed by exploring GitHub via the web
interface and seeing the changes in `latest.md` (the human
file commited).

Additionally, one can clone the `benchmarks` branch and run
`phpbench log` to explore the history in more detail.

Some adjustments to previously added benchmarks were made:

 - Assertions were included in order to track time and memory
   tresholds.
 - The benchmarks are now more surgical, and address the
   concrete validators instead of the whole chain validate.

These changes were made to make benchmarks more isolated, with
the intention of adding chain-related benchmarks separately
in the future.

Author: alganet

.github/workflows/continuous-integration-perf.yml

@@ -0,0 +1,88 @@
+name: Continuous Integration (perf)
+
+on:
+  push:
+    paths-ignore:
+      - 'bin/**'
+      - 'docs/**'
+  pull_request:
+    paths-ignore:
+      - 'bin/**'
+      - 'docs/**'
+  workflow_dispatch:
+    inputs:
+      baseline:
+        description: 'Baseline mode. latest: compare against latest benchmarks; rebaseline: store new baseline.'
+        type: choice
+        default: 'latest'
+        options:
+          - 'latest'
+          - 'rebaseline'
+
+jobs:
+  tests:
+    name: Benchmarks
+    runs-on: ubuntu-latest
+    continue-on-error: true # This job is experimental
+    permissions:
+      contents: write
+      pull-requests: write
+
+    strategy:
+      matrix:
+        php-version:
+          - "8.5"
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: true
+
+      - name: Install PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: ${{ matrix.php-version }}
+          extensions: xdebug
+
+      - name: Install Dependencies
+        run: composer install --prefer-dist ${{ matrix.composer-extra-arguments }}
+
+      - name: Fetch Benchmarks
+        run: |
+          git fetch origin benchmarks
+          mkdir -p .phpbench
+          git checkout origin/benchmarks -- .phpbench || echo "No previous benchmarks found"
+      
+      - name: Run Benchmarks
+        run: |
+          # Baseline does not exist or rebaseline requested. Generate it.
+          if [ -z "$(ls -A .phpbench)" ] || [ "${{ github.event.inputs.baseline || 'latest' }}" = "rebaseline" ]; then
+            vendor/bin/phpbench run --report=aggregate --progress=plain --store --tag=${GITHUB_SHA}
+
+          # On main branch push, update baseline with tolerance for failures.
+          elif [ "${GITHUB_REF}" = "refs/heads/main" ] && [ "${GITHUB_EVENT_NAME}" = "push" ]; then
+            vendor/bin/phpbench run --report=aggregate --progress=plain --store --tag=${GITHUB_SHA} --ref=latest --tolerate-failure
+
+          # On other branches, compare against latest baseline, fails if worse.
+          else 
+            vendor/bin/phpbench run --report=aggregate --progress=plain --store --tag=${GITHUB_SHA} --ref=latest
+          fi
+          
+          # Generate report for human consumption
+          vendor/bin/phpbench report --report=aggregate --ref=latest |
+            tail -n+2 | head -n-2 | tr '+' '|' > report.md
+
+          cat report.md
+
+      - name: Commit Benchmark Results
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        run: |
+          set -euo pipefail
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git checkout benchmarks
+          mv -f report.md latest.md
+          git add .phpbench latest.md
+          git commit -m "Store benchmark results [skip ci]" || echo "No changes to commit"
+          git push origin benchmarks

tests/benchmark/ValidatorBench.php

@@ -18,13 +18,26 @@ class ValidatorBench
     use SmokeTestProvider;
 
     /** @param array<Validator, mixed> $params */
+    #[Bench\ParamProviders(['provideValidatorInput'])]
+    #[Bench\BeforeMethods('setFileUploadMock')]
     #[Bench\Iterations(10)]
-    #[Bench\RetryThreshold(10)]
+    #[Bench\RetryThreshold(5)]
     #[Bench\Revs(5)]
-    #[Bench\ParamProviders(['provideValidatorInput'])]
-    public function benchValidate(array $params): void
+    #[Bench\Warmup(1)]
+    #[Bench\Assert('mode(variant.time.avg) < mode(baseline.time.avg) +/- 10%')]
+    #[Bench\Assert('mode(variant.time.net) < mode(baseline.time.net) +/- 10%')]
+    #[Bench\Assert('mode(variant.mem.peak) < mode(baseline.mem.peak) +/- 10%')]
+    #[Bench\Assert('mode(variant.mem.real) < mode(baseline.mem.real) +/- 10%')]
+    #[Bench\Assert('mode(variant.mem.final) < mode(baseline.mem.final) +/- 10%')]
+    #[Bench\Subject]
+    public function evaluate(array $params): void
     {
         [$v, $input] = $params;
-        $v->validate($input);
+        $v->evaluate($input);
+    }
+
+    public function setFileUploadMock(): void
+    {
+        set_mock_is_uploaded_file_return(true);
     }
 }

tests/feature/SerializableTest.php

@@ -12,7 +12,7 @@
 test('Can be serialized and unserialized', function ($validator, $input): void {
     set_mock_is_uploaded_file_return(true);
     expect(
-        unserialize(serialize($validator))->validate($input)->isValid(),
+        unserialize(serialize($validator))->evaluate($input)->hasPassed,
     )->toBeTrue();
 })->with(fn(): Generator => (new class {
     use SmokeTestProvider {