refactoring: rewrite sync to class (#137)

Refactor to having a single class for syncing Stripe entities - while the class itself is a bit large, it avoids having to do parameter drilling through many methods and actually reduces overall code by quite a bit.

Additionally
- Reduced boilerplate for backfills (generic function)
- Reduced boilerplate for deletions
- Reduced boilerplate for auto-expanding lists
- Migrated from jest to vitest
- Sped up CI
- Config option to enable/disable automatic backfills
- Configurable max connections
- Migrate to pnpm

Author: kevcodez

.env.sample

@@ -23,3 +23,12 @@ STRIPE_API_VERSION="2020-08-27"
 ## - charges.refunds
 ## - subscription.items
 AUTO_EXPAND_LISTS=false
+
+# optional
+# If true, the sync engine will backfill related entities, i.e. when a invoice webhook comes in, it ensures that the customer is present and synced.
+# This ensures foreign key integrity, but comes at the cost of additional queries to the database (and added latency for Stripe calls if the entity is actually missing).
+BACKFILL_RELATED_ENTITIES=true
+
+# optional, default 10
+# Max number of connections for the Postgres connection pool, higher value lead to more concurrent queries, but also more load on the database (connections are expensive)
+MAX_POSTGRES_CONNECTIONS=20

.github/workflows/ci.yml

@@ -9,32 +9,35 @@ on:
 
 jobs:
   test:
-    name: Test / OS ${{ matrix.platform }} / Node ${{ matrix.node }}
-    strategy:
-      fail-fast: false
-      matrix:
-        platform: [ubuntu-24.04]
-        node: ['22']
+    name: Test
+    runs-on: ubuntu-24.04
 
-    runs-on: ${{ matrix.platform }}
+    services:
+      postgres:
+        image: postgres:15
+        ports:
+          - 55432:5432
+        env:
+          POSTGRES_DB: postgres
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
 
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/cache@v4
-        with:
-          path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-node-
+      
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+
       - name: Set up Node
         uses: actions/setup-node@v4
         with:
-          node-version: ${{ matrix.node }}
+          node-version-file: ./.nvmrc
+          cache: pnpm
 
       - name: Set up .env file
         run: |
           touch .env
-          echo DATABASE_URL='postgres://postgres:postgres@127.0.0.1:55432/postgres?sslmode=disable&search_path=stripe' >> .env
+          echo DATABASE_URL='postgres://postgres:postgres@localhost:55432/postgres?sslmode=disable&search_path=stripe' >> .env
           echo NODE_ENV=dev >> .env
           echo STRIPE_SECRET_KEY=sk_test_ >> .env
           echo STRIPE_WEBHOOK_SECRET=whsec_ >> .env
@@ -44,21 +47,29 @@ jobs:
 
       - name: Install dependencies
         run: |
-          npm ci
+          pnpm install --frozen-lockfile
 
       - name: Formatting checks
         run: |
-          npm run format:check
+          pnpm format:check
 
       - name: Lint
         run: |
-          npm run lint
+          pnpm lint
 
       - name: Builds successfully
         run: |
-          npm run build
+          pnpm typecheck
+
+
+      - name: Initialize DB schema
+        run: |
+          docker run --rm \
+            --network="host" \
+            -e PGPASSWORD=postgres \
+            postgres:15 \
+            psql -h localhost -p 55432 -U postgres -d postgres -c 'create schema if not exists "stripe";'
 
       - name: Tests
         run: |
-          docker compose -f ./docker/compose.yml up -d
           npm run test

.github/workflows/release.yml

@@ -1,9 +1,9 @@
 name: release
 
 on:
-  push:
-    branches:
-      - 'main'
+  #push:
+  #  branches:
+  #    - 'main'
   workflow_dispatch:
 
 jobs:
@@ -29,8 +29,7 @@ jobs:
           node-version: ${{ matrix.node }}
 
       - run: |
-          npm clean-install
-          npm run build
+          pnpm install --frozen-lockfile
 
       - name: Run semantic-release
         id: semantic-release

.prettierignore

@@ -1,6 +1,6 @@
 .expo
 .next
 node_modules
-package-lock.json
+pnpm-lock.yaml
 docker*
 Pulumi.*.yaml

.tours/implement-a-webhook.tour

@@ -1,11 +1,14 @@
 # Build step
 FROM node:22-alpine
+
+RUN npm install -g [email protected]
+
 WORKDIR /app
-COPY package.json package-lock.json ./
-RUN npm ci 
+COPY package.json pnpm-lock.yaml ./
+RUN pnpm install --frozen-lockfile
 COPY . /app
-RUN npm run build 
-RUN npm prune --production
+RUN pnpm build
+RUN pnpm prune --production
 
 ## Build step complete, copy to working image
 FROM node:22-alpine

Dockerfile

@@ -175,8 +175,8 @@ The entity type is recognized automatically, based on the prefix.
 
 **Develop**
 
-- `npm run dev` to start the local server
-- `npm run test` to run tests
+- `pnpm dev` to start the local server
+- `pnpm t` to run tests
 
 **Building Docker**
 

README.md

@@ -1,24 +1,24 @@
-import tsParser from "@typescript-eslint/parser";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
-import js from "@eslint/js";
-import { FlatCompat } from "@eslint/eslintrc";
+import tsParser from '@typescript-eslint/parser'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import js from '@eslint/js'
+import { FlatCompat } from '@eslint/eslintrc'
 
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
 const compat = new FlatCompat({
-    baseDirectory: __dirname,
-    recommendedConfig: js.configs.recommended,
-    allConfig: js.configs.all
-});
+  baseDirectory: __dirname,
+  recommendedConfig: js.configs.recommended,
+  allConfig: js.configs.all,
+})
 
 export default [
-    ...compat.extends("plugin:@typescript-eslint/recommended", "plugin:prettier/recommended"),
-    {
-        languageOptions: {
-            parser: tsParser,
-            ecmaVersion: 2022,
-            sourceType: "module",
-        },
+  ...compat.extends('plugin:@typescript-eslint/recommended', 'plugin:prettier/recommended'),
+  {
+    languageOptions: {
+      parser: tsParser,
+      ecmaVersion: 2022,
+      sourceType: 'module',
     },
-];
+  },
+]