Add basic performance testing via Cypress (#8586)

Author: jryans

.github/workflows/element-build-and-test.yaml

@@ -88,6 +88,20 @@ jobs:
             cypress/videos
             cypress/synapselogs
 
+      - name: Store benchmark result
+        if: github.ref == 'refs/heads/develop'
+        uses: matrix-org/github-action-benchmark@jsperfentry-1
+        with:
+          name: Cypress measurements
+          tool: 'jsperformanceentry'
+          output-file-path: cypress/performance/measurements.json
+          # The dashboard is available at https://matrix-org.github.io/matrix-react-sdk/cypress/bench/
+          benchmark-data-dir-path: cypress/bench
+          fail-on-alert: false
+          comment-on-alert: false
+          github-token: ${{ secrets.DEPLOY_GH_PAGES }}
+          auto-push: ${{ github.ref == 'refs/heads/develop' }}
+
   app-tests:
     name: Element Web Integration Tests
     runs-on: ubuntu-latest

.github/workflows/end-to-end-tests.yaml

@@ -40,20 +40,18 @@ jobs:
             test/end-to-end-tests/synapse/installations/consent/homeserver.log
             retention-days: 14
 
-      - name: Download previous benchmark data
-        uses: actions/cache@v1
-        with:
-          path: ./cache
-          key: ${{ runner.os }}-benchmark
-
       - name: Store benchmark result
+        if: github.ref == 'refs/heads/develop'
         uses: matrix-org/github-action-benchmark@jsperfentry-1
         with:
           tool: 'jsperformanceentry'
           output-file-path: test/end-to-end-tests/performance-entries.json
+          # This is the default dashboard path. It's included here anyway to
+          # make the difference from the Cypress variant in
+          # `element-build-and-test.yaml` more obvious.
+          # The dashboard is available at https://matrix-org.github.io/matrix-react-sdk/dev/bench/
+          benchmark-data-dir-path: dev/bench
           fail-on-alert: false
           comment-on-alert: false
-          # Only temporary to monitor where failures occur
-          alert-comment-cc-users: '@gsouquet'
           github-token: ${{ secrets.DEPLOY_GH_PAGES }}
           auto-push: ${{ github.ref == 'refs/heads/develop' }}

.gitignore

@@ -27,3 +27,4 @@ package-lock.json
 # These could have files in them but don't currently
 # Cypress will still auto-create them though...
 /cypress/fixtures
+/cypress/performance

cypress/global.d.ts

@@ -18,6 +18,7 @@ import "matrix-js-sdk/src/@types/global";
 import type { MatrixClient, ClientEvent } from "matrix-js-sdk/src/client";
 import type { RoomMemberEvent } from "matrix-js-sdk/src/models/room-member";
 import type { MatrixDispatcher } from "../src/dispatcher/dispatcher";
+import type PerformanceMonitor from "../src/performance";
 
 declare global {
     // eslint-disable-next-line @typescript-eslint/no-namespace
@@ -27,6 +28,7 @@ declare global {
                 matrixClient?: MatrixClient;
             };
             mxDispatcher: MatrixDispatcher;
+            mxPerformanceMonitor: PerformanceMonitor;
             beforeReload?: boolean; // for detecting reloads
             // Partial type for the matrix-js-sdk module, exported by browser-matrix
             matrixcs: {
@@ -38,7 +40,11 @@ declare global {
     }
 
     interface Window {
-        mxDispatcher: MatrixDispatcher; // to appease the MatrixDispatcher import
+        // to appease the MatrixDispatcher import
+        mxDispatcher: MatrixDispatcher;
+        // to appease the PerformanceMonitor import
+        mxPerformanceMonitor: PerformanceMonitor;
+        mxPerformanceEntryNames: any;
     }
 }
 

cypress/integration/1-register/register.spec.ts

@@ -42,11 +42,15 @@ describe("Registration", () => {
         cy.get("#mx_RegistrationForm_username").type("alice");
         cy.get("#mx_RegistrationForm_password").type("totally a great password");
         cy.get("#mx_RegistrationForm_passwordConfirm").type("totally a great password");
+        cy.startMeasuring("create-account");
         cy.get(".mx_Login_submit").click();
 
         cy.get(".mx_RegistrationEmailPromptDialog button.mx_Dialog_primary").click();
+        cy.stopMeasuring("create-account");
         cy.get(".mx_InteractiveAuthEntryComponents_termsPolicy input").click();
+        cy.startMeasuring("from-submit-to-home");
         cy.get(".mx_InteractiveAuthEntryComponents_termsSubmit").click();
         cy.url().should('contain', '/#/home');
+        cy.stopMeasuring("from-submit-to-home");
     });
 });

cypress/integration/2-login/login.spec.ts

@@ -49,9 +49,11 @@ describe("Login", () => {
 
             cy.get("#mx_LoginForm_username").type(username);
             cy.get("#mx_LoginForm_password").type(password);
+            cy.startMeasuring("from-submit-to-home");
             cy.get(".mx_Login_submit").click();
 
             cy.url().should('contain', '/#/home');
+            cy.stopMeasuring("from-submit-to-home");
         });
     });
 });

cypress/integration/4-create-room/create-room.spec.ts

@@ -54,10 +54,12 @@ describe("Create Room", () => {
             // Fill room address
             cy.get('[label="Room address"]').type("test-room-1");
             // Submit
+            cy.startMeasuring("from-submit-to-room");
             cy.get(".mx_Dialog_primary").click();
         });
 
         cy.url().should("contain", "/#/room/#test-room-1:localhost");
+        cy.stopMeasuring("from-submit-to-room");
         cy.get(".mx_RoomHeader_nametext").contains(name);
         cy.get(".mx_RoomHeader_topic").contains(topic);
     });

cypress/plugins/index.ts

@@ -16,13 +16,15 @@ limitations under the License.
 
 /// <reference types="cypress" />
 
-import { synapseDocker } from "./synapsedocker";
 import PluginEvents = Cypress.PluginEvents;
 import PluginConfigOptions = Cypress.PluginConfigOptions;
+import { performance } from "./performance";
+import { synapseDocker } from "./synapsedocker";
 
 /**
  * @type {Cypress.PluginConfig}
  */
 export default function(on: PluginEvents, config: PluginConfigOptions) {
+    performance(on, config);
     synapseDocker(on, config);
 }

cypress/plugins/performance.ts

@@ -0,0 +1,47 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/// <reference types="cypress" />
+
+import * as path from "path";
+import * as fse from "fs-extra";
+
+import PluginEvents = Cypress.PluginEvents;
+import PluginConfigOptions = Cypress.PluginConfigOptions;
+
+// This holds all the performance measurements throughout the run
+let bufferedMeasurements: PerformanceEntry[] = [];
+
+function addMeasurements(measurements: PerformanceEntry[]): void {
+    bufferedMeasurements = bufferedMeasurements.concat(measurements);
+    return null;
+}
+
+async function writeMeasurementsFile() {
+    try {
+        const measurementsPath = path.join("cypress", "performance", "measurements.json");
+        await fse.outputJSON(measurementsPath, bufferedMeasurements, {
+            spaces: 4,
+        });
+    } finally {
+        bufferedMeasurements = [];
+    }
+}
+
+export function performance(on: PluginEvents, config: PluginConfigOptions) {
+    on("task", { addMeasurements });
+    on("after:run", writeMeasurementsFile);
+}

cypress/plugins/synapsedocker/index.ts

@@ -201,7 +201,7 @@ async function synapseStop(id: string): Promise<void> {
     synapses.delete(id);
 
     console.log(`Stopped synapse id ${id}.`);
-    // cypres deliberately fails if you return 'undefined', so
+    // cypress deliberately fails if you return 'undefined', so
     // return null to signal all is well and we've handled the task.
     return null;
 }