feat: Run E2E tests aganist Platform Pt.1 (supabase#41032)

* added packages for creating projects

* updated scripts

* remove ami version

* cleaned up common

* updated tests

* refactored helpers

* updated env

* updated config

* updated to reference env

* updated global setup

* updated type logic and scripts

* added mocking of hcaptcha

* added log statements

* updated local env

* update env file

* updated env vars

* updated logging

* updated to remove check

* updated print and project names

* updated helpers

* updated url

* updated setup

* updated storage helpers to account for listing files

* updated setup and tests

* updated timeout only for setup

* updated helper to account for different api response

* added ignores for tests

* updated lock file

* updated database spec to add exact

* updated timeouts

* removed check for table grid footer

* updated test runner

* updated is_platform

* updated playwright config

* updated worker settings

* removed dotenvx

* updated README

* updated to remove comment

* Update e2e/studio/scripts/common/retriedFetch.ts

Co-authored-by: Charis <[email protected]>

---------

Co-authored-by: Charis <[email protected]>

Author: awaseem

e2e/studio/.env.local.platform.example

@@ -0,0 +1,14 @@
+
+# Copy and paste this file and rename it to .env.local
+
+# Required for self hosted tests
+# Mise infra running
+STUDIO_URL=http://localhost:8082
+API_URL=http://localhost:8080
+
+# Required for platform tests
+IS_PLATFORM=true
+ORG_SLUG=
+SUPA_PAT=
+EMAIL=
+PASSWORD=

e2e/studio/.env.local.example e2e/studio/.env.local.self-hosted.examplee2e/studio/.env.local.example renamed to e2e/studio/.env.local.self-hosted.example

@@ -1,9 +1,9 @@
 
 # Copy and paste this file and rename it to .env.local
 
+# Required for self hosted tests
 STUDIO_URL=http://localhost:8082
 API_URL=http://127.0.0.1:54321
-IS_PLATFORM=false
 
-# Used to run e2e tests against vercel previews
-VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO=
+# Required for platform tests
+IS_PLATFORM=false

e2e/studio/README.md

@@ -2,10 +2,27 @@
 
 ## Set up
 
+### Prerequisites
+
+#### For Self-Hosted Tests
+
+- Nothing is required, running with IS_PLATFORM=false should run the tests locally with a self hosted docker container
+
+#### For Platform Tests
+
+1. **Create a platform account** with an email and password, these auths are used for the test
+2. **Create an organization** on the platform, this can be done if run locally through `mise fullstack`
+3. **Generate a Personal Access Token (PAT)** for API access
+4. Configure the environment variables below
+
+### Configure Environment
+
 ```bash
 cp .env.local.example .env.local
 ```
 
+Edit `.env.local` and set the appropriate values based on your test environment (see Environment Variables section below).
+
 ### Install the playwright browser
 
 ⚠️ This should be done in the `e2e/studio` directory
@@ -18,9 +35,45 @@ pnpm exec playwright install
 
 ### Environment Variables
 
-Some tests require specific environment variables to be set. If these are not set, the tests will be automatically skipped:
+Configure your tests by setting the following environment variables in `.env.local`. We have examples of what required on self hosted and platform:
+
+#### Core Configuration
+
+- **`STUDIO_URL`**: The URL where Studio is running (default: `http://localhost:8082`)
+- **`API_URL`**: The Supabase API endpoint (default: `https://localhost:8080`)
+- **`IS_PLATFORM`**: Set to `true` for platform tests, `false` for self-hosted (default: `false`)
+  - When `true`: Tests run serially (1 worker) due to API rate limits
+  - When `false`: Tests run in parallel (5 workers)
+
+#### Authentication (Required for Platform Tests)
+
+⚠️ **Before running platform tests, you must create an account with an email, password, and organization on the platform you're testing.**
+
+- **`EMAIL`**: Your platform account email (required for authentication)
+- **`PASSWORD`**: Your platform account password (required for authentication)
+- **`PROJECT_REF`**: Project reference (optional, will be auto-created if not provided)
+
+When both `EMAIL` and `PASSWORD` are set, authentication is automatically enabled. HCaptcha is mocked during test setup.
+
+#### Platform-Specific Variables (Required when `IS_PLATFORM=true`)
+
+- **`ORG_SLUG`**: Organization slug (default: `default`)
+- **`SUPA_REGION`**: Supabase region (default: `us-east-1`)
+- **`SUPA_PAT`**: Personal Access Token for API authentication (default: `test`)
+- **`BRANCH_NAME`**: Name for the test branch/project (default: `e2e-test-local`)
+
+#### Optional Variables
 
 - **`OPENAI_API_KEY`**: Required for the AI Assistant test (`assistant.spec.ts`). Without this variable, the assistant test will be skipped.
+- **`VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO`**: Bypass token for Vercel protection (default: `false`)
+
+#### Setup Commands Based on Configuration
+
+The test setup automatically runs different commands based on your environment:
+
+- **Platform + Localhost** (`IS_PLATFORM=true` and `STUDIO_URL=localhost`): Runs `pnpm run e2e:setup:platform`
+- **Platform + Remote** (`IS_PLATFORM=true` and remote `STUDIO_URL`): No web server setup
+- **Self-hosted** (`IS_PLATFORM=false`): Runs `pnpm run e2e:setup:selfhosted`
 
 ---
 

e2e/studio/env.config.ts

@@ -1,21 +1,39 @@
+import dotenv from 'dotenv'
 import path from 'path'
 
+// Load .env.local before reading process.env
+dotenv.config({
+  path: path.resolve(import.meta.dirname, '.env.local'),
+  override: true,
+})
+
 const toBoolean = (value?: string) => {
   if (value == null) return false
   const normalized = value.trim().toLowerCase()
   return normalized === 'true'
 }
 
 export const env = {
-  STUDIO_URL: process.env.STUDIO_URL,
-  API_URL: process.env.API_URL || 'https://api.supabase.green',
-  AUTHENTICATION: toBoolean(process.env.AUTHENTICATION),
+  STUDIO_URL: process.env.STUDIO_URL || 'http://localhost:8082',
+  API_URL: process.env.API_URL || 'https://localhost:8080',
+
+  IS_PLATFORM: toBoolean(process.env.IS_PLATFORM || 'false'),
   EMAIL: process.env.EMAIL,
   PASSWORD: process.env.PASSWORD,
-  PROJECT_REF: process.env.PROJECT_REF || 'default',
-  IS_PLATFORM: process.env.IS_PLATFORM || 'false',
+  PROJECT_REF: process.env.PROJECT_REF || undefined,
+
   VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO:
     process.env.VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO || 'false',
+  ORG_SLUG: process.env.ORG_SLUG || 'default',
+  SUPA_REGION: process.env.SUPA_REGION || 'us-east-1',
+  SUPA_PAT: process.env.SUPA_PAT || 'test',
+
+  BRANCH_NAME: process.env.BRANCH_NAME || `e2e-test-local`,
+
+  AUTHENTICATION: Boolean(process.env.EMAIL && process.env.PASSWORD),
+
+  IS_APP_RUNNING_ON_LOCALHOST:
+    process.env.STUDIO_URL?.includes('localhost') || process.env.STUDIO_URL?.includes('127.0.0.1'),
 }
 
 export const STORAGE_STATE_PATH = path.join(import.meta.dirname, './playwright/.auth/user.json')