Feat/add custom exit codes

.github/workflows/e2e-validation.yml

@@ -218,6 +218,8 @@ jobs:
     runs-on: ubuntu-latest
     needs: [define-test-matrix, launch-ec2-node]
     if: always() && (needs.define-test-matrix.result == 'success') && (inputs.environment != 'ec2-node' || needs.launch-ec2-node.result == 'success')
+    outputs:
+      has-flaky-failures: ${{ steps.set-outputs.outputs.has-flaky-failures }}
     strategy:
       fail-fast: true
       matrix:
@@ -334,6 +336,14 @@ jobs:
               echo "Cypress Test task succeeded!"
               break
             fi
+
+            # Check if this is a flaky failure (exit code 53)
+            if [ "$exit_code" = "53" ]; then
+              echo "Flaky failure detected (exit code 53) - marking as flaky failure"
+              echo "flaky_failure=true" >> "$GITHUB_ENV"
+              exit 53
+            fi
+
             n=$((n+1))
             echo "Attempt $n/$max_retries failed with exit code $exit_code! Retrying..."
           done
@@ -351,6 +361,16 @@ jobs:
           path: app/web/cypress/videos/**/*.mp4
           retention-days: 5
 
+      - name: Set job outputs
+        id: set-outputs
+        if: always()
+        run: |
+          if [ "${{ env.flaky_failure }}" = "true" ]; then
+            echo "has-flaky-failures=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "has-flaky-failures=false" >> "$GITHUB_OUTPUT"
+          fi
+
       - name: Check Test Results
         if: failure()
         run: exit 1
@@ -434,7 +454,10 @@ jobs:
               break
             fi
           done
-          if [ "$has_artifacts" = true ] && [ "${{ github.ref_name }}" = "main" ]; then
+
+          # Only send FireHydrant alert for non-flaky failures on main branch
+          if [ "$has_artifacts" = true ] && [ "${{ github.ref_name }}" = "main" ] && [ "${{ needs.cypress-tests.outputs.has-flaky-failures }}" != "true" ]; then
+            echo "Sending FireHydrant alert for real test failures"
             curl --location "${{ secrets.FIREHYDRANT_WEBHOOK_URL }}" \
               --header "Content-Type: application/json" \
               --data "{
@@ -450,6 +473,8 @@ jobs:
                   \"service:github\"
                 ]
               }"
+          elif [ "${{ needs.cypress-tests.outputs.has-flaky-failures }}" = "true" ]; then
+            echo "Skipping FireHydrant alert - flaky failure detected (exit code 53)"
           fi
 
       - name: Send Slack notification with deployment error
@@ -466,12 +491,22 @@ jobs:
         if: ${{ inputs.environment != 'ec2-node' || (inputs.environment == 'ec2-node' && needs.launch-ec2-node.result != 'failure') }}
         run: |
           failed_tests="${{ steps.failed-tests.outputs.failed_tests }}"
-          if [ -n "$failed_tests" ]; then
+          is_flaky="${{ needs.cypress-tests.outputs.has-flaky-failures }}"
+
+          if [ "$is_flaky" = "true" ]; then
+            # Flaky failure - send different message
+            curl -X POST \
+            --header 'Content-type: application/json' \
+            --data "{\"text\": \":si: Cypress E2E Test Flaked for ${{ inputs.environment }} (exit code 53): <https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID|:test_tube: Link>\n\`\`\`Flaky test failure detected - not paging just notifying\`\`\`\"}" \
+            ${{ secrets.SLACK_WEBHOOK_URL }}
+          elif [ -n "$failed_tests" ]; then
+            # Regular test failure
             curl -X POST \
             --header 'Content-type: application/json' \
             --data "{\"text\": \":si: Failed Cypress E2E Test for ${{ inputs.environment }}: <https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID|:test_tube: Link>\n\`\`\`Failed tests: $failed_tests\`\`\`\"}" \
             ${{ secrets.SLACK_WEBHOOK_URL }}
           else
+            # Generic failure
             curl -X POST \
             --header 'Content-type: application/json' \
             --data "{\"text\": \":si: Failed Cypress E2E Test for ${{ inputs.environment }}: <https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID|:test_tube: Link>\"}" \

app/web/cypress.config.ts

@@ -2,6 +2,8 @@ import path from 'path'
 import { defineConfig } from 'cypress'
 import vitePreprocessor from 'cypress-vite'
 
+const FLAKY_EXIT_CODE = 53
+
 export default defineConfig({
   e2e: {
     injectDocumentDomain: true,
@@ -11,13 +13,45 @@ export default defineConfig({
         vitePreprocessor(
           path.resolve('./vite.cypress.ts'),
         )
-      ),
-        on('task', {
-          log(message) {
-            console.log(message)
-            return null
-          }
-        })
+      );
+
+      on('task', {
+        log(message) {
+          console.log(message)
+          return null
+        },
+        flakyFailure() {
+          console.log('Flaky failure detected - will fail test and exit with code', FLAKY_EXIT_CODE)
+          // Set a flag that can be checked in after:run hook
+          process.env.FLAKY_FAILURE_DETECTED = 'true'
+          // Throw an error to fail the test
+          throw new Error('Simulated flaky failure')
+        }
+      });
+
+      on('after:run', (results: any) => {
+        // Check if flaky failure was explicitly triggered
+        if (process.env.FLAKY_FAILURE_DETECTED === 'true') {
+          console.log('Flaky failure was detected during test run - exiting with code', FLAKY_EXIT_CODE)
+          // Use setImmediate to allow Cypress to finish cleanup before exiting
+          setImmediate(() => process.exit(FLAKY_EXIT_CODE))
+          return
+        }
+
+        // Check for Auth0-related failures in test results
+        const hasAuth0Failures = results.runs?.some((run: any) => 
+          run.tests?.some((test: any) => 
+            test.displayError?.includes('Auth0') || 
+            test.title?.includes('auth0') ||
+            test.err?.message?.includes('Auth0')
+          )
+        );
+
+        if (hasAuth0Failures) {
+          console.log('Detected Auth0-related test failures - exiting with code', FLAKY_EXIT_CODE)
+          setImmediate(() => process.exit(FLAKY_EXIT_CODE))
+        }
+      });
     },
 
     // Hotfix, needs amended

app/web/cypress/e2e/workspace-management/workspace-dashboard.cy.ts

@@ -15,9 +15,20 @@ Cypress._.times(SI_CYPRESS_MULTIPLIER, () => {
       cy.visit("/");
     });
 
+    it("simulate_flaky_failure", () => {
+      // Temporary test to simulate flaky failure for workflow testing
+      cy.task('flakyFailure');
+    });
+
     it("dashboard_redirect", () => {
 
-      cy.loginToAuth0(AUTH0_USERNAME, AUTH0_PASSWORD);
+      try {
+        cy.loginToAuth0(AUTH0_USERNAME, AUTH0_PASSWORD);
+      } catch (_err) {
+        // flaky failures should not ping us
+        cy.task('flakyFailure');
+        return;
+      }
 
       // Go to the Synthetic User's Dashboard
       cy.visit(AUTH_PORTAL_URL + '/dashboard')

app/web/cypress/support/commands.ts

@@ -107,7 +107,13 @@ Cypress.Commands.add('basicLogin', () => {
   const SI_WORKSPACE_ID = Cypress.env('VITE_SI_WORKSPACE_ID') || import.meta.env.VITE_SI_WORKSPACE_ID;
   const UUID = Cypress.env('VITE_UUID') || import.meta.env.VITE_UUID || "local";
 
-  cy.loginToAuth0(AUTH0_USERNAME, AUTH0_PASSWORD);
+  try {
+    cy.loginToAuth0(AUTH0_USERNAME, AUTH0_PASSWORD);
+  } catch (_err) {
+    // flaky failures should not ping us
+    cy.task('flakyFailure');
+    return;
+  }
   cy.visit({
     url:AUTH_API_URL + '/workspaces/' + SI_WORKSPACE_ID + '/go',
     failOnStatusCode: false

app/web/cypress/ux/tile-view/get-tiles.cy.ts

@@ -12,11 +12,15 @@ const AUTH_API_URL = Cypress.env('VITE_AUTH_API_URL') || import.meta.env.VITE_AU
 const AUTH_PORTAL_URL = Cypress.env('VITE_AUTH_PORTAL_URL') || import.meta.env.VITE_AUTH_PORTAL_URL;
 
 describe("web", () => {
-  beforeEach(function () {
-    cy.loginToAuth0(AUTH0_USERNAME, AUTH0_PASSWORD);
-  });
 
   it("get_ptlw_tiles", () => {
+    try {
+      cy.loginToAuth0(AUTH0_USERNAME, AUTH0_PASSWORD);
+    } catch (_err) {
+      // flaky failures should not ping us
+      cy.task('flakyFailure');
+      return;
+    }
     // Go to the Synthetic User's Dashboard
     cy.visit(AUTH_PORTAL_URL + '/dashboard')
     cy.sendPosthogEvent(Cypress.currentTest.titlePath.join("/"), "test_uuid", UUID);