feat: Shared SST code for example deployments (#2270)

Co-authored-by: Valter Balegas <[email protected]>

Author: msfstef

.github/workflows/deploy_all_examples.yml

@@ -10,11 +10,13 @@ concurrency:
 jobs:
   deploy-examples:
     name: Deploy All Examples to Production
-    environment: 'Production'
+    environment: "Production"
     runs-on: ubuntu-latest
 
     env:
-      DEPLOY_ENV: 'production'
+      DEPLOY_ENV: "production"
+      SHARED_INFRA_VPC_ID: ${{ vars.SHARED_INFRA_VPC_ID }}
+      SHARED_INFRA_CLUSTER_ARN: ${{ vars.SHARED_INFRA_CLUSTER_ARN }}
       CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
       CLOUDFLARE_DEFAULT_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_DEFAULT_ACCOUNT_ID }}
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

.github/workflows/deploy_examples.yml

@@ -17,6 +17,8 @@ jobs:
 
     env:
       DEPLOY_ENV: ${{ github.event_name == 'push' && 'production' || format('pr-{0}', github.event.number) }}
+      SHARED_INFRA_VPC_ID: ${{ vars.SHARED_INFRA_VPC_ID }}
+      SHARED_INFRA_CLUSTER_ARN: ${{ vars.SHARED_INFRA_CLUSTER_ARN }}
       CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
       CLOUDFLARE_DEFAULT_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_DEFAULT_ACCOUNT_ID }}
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -53,74 +55,86 @@ jobs:
         working-directory: ./examples/yjs
         run: |
           pnpm sst deploy --stage ${{ env.DEPLOY_ENV }}
+          code=$!
           if [ -f ".sst/outputs.json" ]; then
             yjs=$(jq -r '.website' .sst/outputs.json)
             echo "yjs=$yjs" >> $GITHUB_ENV
           else
             echo "sst outputs file not found. Exiting."
             exit 123
           fi
+          exit $code
 
       - name: Deploy Linearlite Read Only
         working-directory: ./examples/linearlite-read-only
         run: |
           pnpm sst deploy --stage ${{ env.DEPLOY_ENV }}
+          code=$!
           if [ -f ".sst/outputs.json" ]; then
             linearlite_read_only=$(jq -r '.website' .sst/outputs.json)
             echo "linearlite_read_only=$linearlite_read_only" >> $GITHUB_ENV
           else
             echo "sst outputs file not found. Exiting."
             exit 123
           fi
+          exit $code
 
       - name: Deploy Write Patterns example
         working-directory: ./examples/write-patterns
         run: |
           pnpm --filter @electric-sql/client --filter @electric-sql/experimental --filter @electric-sql/react run build
           pnpm sst deploy --stage ${{ env.DEPLOY_ENV }}
+          code=$!
           if [ -f ".sst/outputs.json" ]; then
             writes=$(jq -r '.website' .sst/outputs.json)
             echo "writes=$writes" >> $GITHUB_ENV
           else
             echo "sst outputs file not found. Exiting."
             exit 123
           fi
+          exit $code
 
       - name: Deploy NextJs example
         working-directory: ./examples/nextjs
         run: |
           pnpm sst deploy --stage ${{ env.DEPLOY_ENV }}
+          code=$!
           if [ -f ".sst/outputs.json" ]; then
             nextjs=$(jq -r '.website' .sst/outputs.json)
             echo "nextjs=$nextjs" >> $GITHUB_ENV
           else
             echo "sst outputs file not found. Exiting."
             exit 123
           fi
+          exit $code
 
       - name: Deploy TODO App example
         working-directory: ./examples/todo-app
         run: |
           pnpm sst deploy --stage ${{ env.DEPLOY_ENV }}
+          code=$!
           if [ -f ".sst/outputs.json" ]; then
             todoapp=$(jq -r '.website' .sst/outputs.json)
             echo "todoapp=$todoapp" >> $GITHUB_ENV
           else
             echo "sst outputs file not found. Exiting."
             exit 123
           fi
+          exit $code
 
       - name: Deploy proxy-auth example
         working-directory: ./examples/proxy-auth
         run: |
           pnpm sst deploy --stage ${{ env.DEPLOY_ENV }}
+          code=$!
           if [ -f ".sst/outputs.json" ]; then
             auth=$(jq -r '.website' .sst/outputs.json)
             echo "auth=$auth" >> $GITHUB_ENV
           else
             echo "sst outputs file not found. Exiting."
             exit 123
           fi
+          exit $code
 
       - name: Add comment to PR
         if: github.event_name == 'pull_request'

.github/workflows/teardown_examples_pr_stack.yml

@@ -13,6 +13,18 @@ jobs:
     name: Teardown Examples PR stack
     environment: Pull request
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        example:
+          [
+            "yjs",
+            "linearlite-read-only",
+            "write-patterns",
+            "nextjs",
+            "todo-app",
+            "proxy-auth",
+          ]
 
     env:
       DEPLOY_ENV: ${{ github.event_name == 'push' && 'production' || format('pr-{0}', github.event.number) }}
@@ -45,15 +57,8 @@ jobs:
           restore-keys: |
             sst-cache-${{ runner.os }}
 
-      - name: Remove Linearlite
-        working-directory: examples/linearlite-read-only
-        run: |
-          export PR_NUMBER=${{ github.event.number }}
-          echo "Removing stage pr-$PR_NUMBER"
-          pnpm sst remove --stage "pr-$PR_NUMBER"
-
-      - name: Remove NextJs example
-        working-directory: examples/nextjs
+      - name: Remove ${{ matrix.example }} example
+        working-directory: ./examples/${{ matrix.example }}
         run: |
           export PR_NUMBER=${{ github.event.number }}
           echo "Removing stage pr-$PR_NUMBER"

.github/workflows/ts_test.yml

@@ -155,6 +155,7 @@ jobs:
           cache: pnpm
       - run: pnpm install --frozen-lockfile
       - run: pnpm -r --filter "$(jq '.name' -r package.json)^..." build
+      - run: pnpm --if-present run prepare
       - run: pnpm --if-present run typecheck
       - run: pnpm --if-present run build
       - run: pnpm --if-present run test

examples/.shared/.eslintrc.cjs

@@ -0,0 +1,35 @@
+module.exports = {
+  env: {
+    browser: true,
+    es2021: true,
+    node: true,
+  },
+  extends: [
+    `eslint:recommended`,
+    `plugin:@typescript-eslint/recommended`,
+    `plugin:prettier/recommended`,
+  ],
+  parserOptions: {
+    ecmaVersion: 2022,
+    requireConfigFile: false,
+    sourceType: `module`,
+    ecmaFeatures: {
+      jsx: true,
+    },
+  },
+  parser: `@typescript-eslint/parser`,
+  plugins: [`prettier`],
+  rules: {
+    quotes: [`error`, `backtick`],
+    'no-unused-vars': `off`,
+    '@typescript-eslint/no-unused-vars': [
+      `error`,
+      {
+        argsIgnorePattern: `^_`,
+        varsIgnorePattern: `^_`,
+        caughtErrorsIgnorePattern: `^_`,
+      },
+    ],
+  },
+  ignorePatterns: [`**/node_modules/**`, `.eslintrc.cjs`],
+}

examples/.shared/.prettierrc

@@ -0,0 +1,6 @@
+{
+  "semi": false,
+  "singleQuote": true,
+  "tabWidth": 2,
+  "trailingComma": "es5"
+}

examples/.shared/lib/database.ts

@@ -0,0 +1,107 @@
+import { execSync } from 'node:child_process'
+import { createNeonDb, getNeonConnectionString } from './neon'
+
+async function addDatabaseToElectric({
+  dbUri,
+}: {
+  dbUri: string
+}): Promise<{ id: string; source_secret: string }> {
+  const adminApi = process.env.ELECTRIC_ADMIN_API
+  const teamId = process.env.ELECTRIC_TEAM_ID
+
+  if (!adminApi || !teamId) {
+    throw new Error(`ELECTRIC_ADMIN_API or ELECTRIC_TEAM_ID is not set`)
+  }
+
+  const adminApiTokenId = process.env.ELECTRIC_ADMIN_API_TOKEN_ID
+  const adminApiTokenSecret = process.env.ELECTRIC_ADMIN_API_TOKEN_SECRET
+  if (!adminApiTokenId || !adminApiTokenSecret) {
+    throw new Error(
+      `ADMIN_API_TOKEN_CLIENT_ID or ADMIN_API_TOKEN_CLIENT_SECRET is not set`
+    )
+  }
+
+  const result = await fetch(`${adminApi}/v1/sources`, {
+    method: `PUT`,
+    headers: {
+      'Content-Type': `application/json`,
+      'CF-Access-Client-Id': adminApiTokenId,
+      'CF-Access-Client-Secret': adminApiTokenSecret,
+    },
+    body: JSON.stringify({
+      database_url: dbUri,
+      region: `us-east-1`,
+      team_id: teamId,
+    }),
+  })
+
+  if (!result.ok) {
+    throw new Error(
+      `Could not add database to Electric (${result.status}): ${await result.text()}`
+    )
+  }
+
+  return await result.json()
+}
+
+function applyMigrations(
+  dbUri: string,
+  migrationsDir: string = `./db/migrations`
+) {
+  execSync(`pnpm exec pg-migrations apply --directory ${migrationsDir}`, {
+    env: {
+      ...process.env,
+      DATABASE_URL: dbUri,
+    },
+  })
+}
+
+export function createDatabaseForCloudElectric({
+  dbName,
+  migrationsDirectory,
+}: {
+  dbName: string
+  migrationsDirectory: string
+}) {
+  const neonProjectId = process.env.NEON_PROJECT_ID
+  if (!neonProjectId) {
+    throw new Error(`NEON_PROJECT_ID is not set`)
+  }
+
+  const project = neon.getProjectOutput({
+    id: neonProjectId,
+  })
+  const { ownerName, dbName: resultingDbName } = createNeonDb({
+    projectId: project.id,
+    branchId: project.defaultBranchId,
+    dbName,
+  })
+
+  const databaseUri = getNeonConnectionString({
+    project,
+    roleName: ownerName,
+    databaseName: resultingDbName,
+    pooled: false,
+  })
+  const pooledDatabaseUri = getNeonConnectionString({
+    project,
+    roleName: ownerName,
+    databaseName: resultingDbName,
+    pooled: true,
+  })
+
+  if (migrationsDirectory) {
+    databaseUri.apply((uri) => applyMigrations(uri, migrationsDirectory))
+  }
+
+  const electricInfo = databaseUri.apply((dbUri) =>
+    addDatabaseToElectric({ dbUri })
+  )
+
+  return {
+    sourceId: electricInfo.id,
+    sourceSecret: electricInfo.source_secret,
+    databaseUri,
+    pooledDatabaseUri,
+  }
+}

examples/.shared/lib/infra.ts

@@ -0,0 +1,17 @@
+export function getSharedCluster(serviceName: string): sst.aws.Cluster {
+  const sharedInfraVpcId = process.env.SHARED_INFRA_VPC_ID
+  const sharedInfraClusterArn = process.env.SHARED_INFRA_CLUSTER_ARN
+  if (!sharedInfraVpcId || !sharedInfraClusterArn) {
+    throw new Error(
+      `SHARED_INFRA_VPC_ID or SHARED_INFRA_CLUSTER_ARN is not set`
+    )
+  }
+
+  return sst.aws.Cluster.get(`${serviceName}-cluster`, {
+    id: sharedInfraClusterArn,
+    vpc: sst.aws.Vpc.get(`${serviceName}-vpc`, sharedInfraVpcId),
+  })
+}
+
+export const isProduction = () =>
+  $app.stage.toLocaleLowerCase() === `production`