Postgres Bucket Storage Demo

CHANGELOG.md

@@ -1,5 +1,9 @@
 # PowerSync Self Hosted Example
 
+## 2025-01-10
+
+- Added demo for using Postgres as the bucket storage.
+
 ## 2024-11-27
 
 - Updated `journeyapps/powersync-service:latest` image specifier which will target `>v.1.0.0` of the Docker image. This version of the Docker image contains support for MongoDB (alpha), MySQL (alpha) and the new modular architecture.

README.md

@@ -34,6 +34,10 @@ This repository contains basic demonstrations in the `demos` folder.
 
   - See the README for instructions.
 
+- [Node.js (Postgres + Postgres Bucket Storage)](./demos/nodejs-postgres-bucket-storage/README.md)
+
+  - This can be started from the repo root with `docker compose -f demos/nodejs-postgres-bucket-storage/docker-compose.yaml up`
+
 # Config
 
 The configuration can be modified to match other project topologies.

config/powersync.yaml

@@ -68,6 +68,7 @@ replication:
 storage:
   type: mongodb
   uri: !env PS_MONGO_URI
+
   # Use these if authentication is required. The user should have `readWrite` and `dbAdmin` roles
   # username: my-mongo-user
   # password: my-password

demos/nodejs-custom-checkpoints/README.md

@@ -11,7 +11,8 @@ The `.env` file contains default configuration for the services. Reference this
 Ensure you have authenticated with our Docker Image repository. Please reach out to support for an access token.
 
 ```bash
-docker login [email protected] -u user
+# the value for user doesn't matter
+docker login container-registry.journeyapps.com -u user 
 ```
 
 This demo can be started by running the following in this demo directory

demos/nodejs-postgres-bucket-storage/.env

@@ -0,0 +1,30 @@
+# ==================== Postgres credentials ================================
+PG_DATABASE_NAME=postgres
+PG_DATABASE_PORT=5432
+PG_DATABASE_USER=postgres
+PG_DATABASE_PASSWORD=postgres
+PS_DATA_SOURCE_URI=postgres://${PG_DATABASE_USER}:${PG_DATABASE_PASSWORD}@pg-db:${PG_DATABASE_PORT}/${PG_DATABASE_NAME}
+
+
+PG_STORAGE_DATABASE_NAME=postgres
+PG_STORAGE_DATABASE_PORT=5431
+PG_STORAGE_DATABASE_USER=postgres
+PG_STORAGE_DATABASE_PASSWORD=postgres
+PS_STORAGE_SOURCE_URI=postgres://${PG_STORAGE_DATABASE_USER}:${PG_STORAGE_DATABASE_PASSWORD}@pg-storage:${PG_STORAGE_DATABASE_PORT}/${PG_STORAGE_DATABASE_NAME}
+
+# ==================== Demo config =========================================
+DEMO_BACKEND_PORT=6060
+DEMO_BACKEND_DATABASE_TYPE=postgres
+DEMO_BACKEND_DATABASE_URI=postgres://${PG_DATABASE_USER}:${PG_DATABASE_PASSWORD}@pg-db:${PG_DATABASE_PORT}/${PG_DATABASE_NAME}
+# The front-end demo application is accessible at this port on the host machine
+DEMO_CLIENT_PORT=3039
+PS_JWKS_URL=http://demo-backend:${DEMO_BACKEND_PORT}/api/auth/keys
+
+# These can be generated by following the instructions in the `key-generator` folder
+# A temporary key will be used if these are not specified
+DEMO_JWKS_PUBLIC_KEY=
+DEMO_JWKS_PRIVATE_KEY=
+
+# ==================== PowerSync variables ====================
+# The PowerSync API is accessible via this port
+PS_PORT=8080

demos/nodejs-postgres-bucket-storage/README.md

@@ -0,0 +1,30 @@
+# JavaScript PowerSync + Postgres Bucket Storage
+
+This is a demo for using Postgres as the sync bucket storage driver with PowerSync.
+
+Separate Postgres servers are required for the bucket storage and replication data source.
+
+## Running
+
+The `.env` file contains default configuration for the services. Reference this to connect to any services locally.
+
+This demo can be started by running the following in this demo directory
+
+```bash
+docker compose up
+```
+
+or in the root directory run
+
+```bash
+docker compose -f demos/nodejs-postgres-bucket-storage/docker-compose.yaml up
+```
+
+The frontend can be accessed at `http://localhost:3039` in a browser.
+
+The Postgres databases can be accessed at the following URIs
+
+Application data: `postgres://postgres:postgres@localhost:5432/postgres`
+
+bucket storage: `postgres://postgres:postgres@localhost:5431/postgres`
+Bucket storage tables are located in the `powersync` schema.

demos/nodejs-postgres-bucket-storage/config/powersync.yaml

@@ -0,0 +1,68 @@
+# yaml-language-server: $schema=../schema/schema.json
+
+# Note that this example uses YAML custom tags for environment variable substitution.
+# Using `!env [variable name]` will substitute the value of the environment variable named
+# [variable name].
+#
+# Only environment variables with names starting with `PS_` can be substituted.
+#
+# e.g. With the environment variable `export PS_MONGO_URI=mongodb://localhost:27017`
+# and YAML code:
+#  uri: !env PS_MONGO_URI
+# The YAML will resolve to:
+#  uri: mongodb://localhost:27017
+#
+# If using VS Code see the `.vscode/settings.json` definitions which define custom tags.
+
+# migrations:
+#   # Migrations run automatically by default.
+#   # Setting this to true will skip automatic migrations.
+#   # Migrations can be triggered externally by altering the container `command`.
+#   disable_auto_migration: true
+
+# Settings for telemetry reporting
+# See https://docs.powersync.com/self-hosting/telemetry
+telemetry:
+  # Opt out of reporting anonymized usage metrics to PowerSync telemetry service
+  disable_telemetry_sharing: false
+
+# Settings for source database replication
+replication:
+  # Specify database connection details
+  # Note only 1 connection is currently supported
+  # Multiple connection support is on the roadmap
+  connections:
+    - type: postgresql
+      # The PowerSync server container can access the Postgres DB via the DB's service name.
+      # In this case the hostname is pg-db
+
+      uri: !env PS_DATA_SOURCE_URI
+      # SSL settings
+      sslmode: disable # 'verify-full' (default) or 'verify-ca' or 'disable'
+
+# Connection settings for sync bucket storage
+storage:
+  type: postgresql
+  # This accepts the same parameters as a postgres data source connection
+  uri: !env PS_STORAGE_SOURCE_URI
+  sslmode: disable # 'verify-full' (default) or 'verify-ca' or 'disable'
+
+# The port which the PowerSync API server will listen on
+port: !env PS_PORT
+
+# Specify sync rules
+sync_rules:
+  path: sync_rules.yaml
+
+# Client (application end user) authentication settings
+client_auth:
+  # JWKS URIs can be specified here
+  jwks_uri: !env PS_JWKS_URL
+
+  # JWKS audience
+  audience: ["powersync-dev", "powersync"]
+
+api:
+  tokens:
+    # These tokens are used for local admin API route authentication
+    - use_a_better_token_in_production

demos/nodejs-postgres-bucket-storage/config/sync_rules.yaml

@@ -0,0 +1,13 @@
+# yaml-language-server: $schema=https://unpkg.com/@powersync/service-sync-rules@latest/schema/sync_rules.json
+#
+# See Documentation for more information:
+# https://docs.powersync.com/usage/sync-rules
+#
+# Note that changes to this file are not watched.
+# The service needs to be restarted for changes to take effect.
+
+bucket_definitions:
+  global:
+    data:
+      - SELECT * FROM lists
+      - SELECT * FROM todos

demos/nodejs-postgres-bucket-storage/docker-compose.yaml

@@ -0,0 +1,49 @@
+# Include syntax requires Docker compose > 2.20.3
+# https://docs.docker.com/compose/release-notes/#2203
+include:
+  # Demo NodeJS backend server and front-end web client
+  - path: ../nodejs/ps-nodejs-demo.yaml
+
+services:
+  # Extend PowerSync with Mongo and Postgres healthchecks
+  powersync:
+    extends:
+      file: ../../services/powersync.yaml
+      service: powersync
+    depends_on:
+      pg-db:
+        condition: service_healthy
+      pg-storage:
+        condition: service_healthy
+    volumes:
+      - ./config:/config
+    environment:
+      PS_STORAGE_SOURCE_URI: ${PS_STORAGE_SOURCE_URI}
+  pg-db:
+    extends:
+      file: ../../services/postgres.yaml
+      service: pg-db
+    volumes:
+      - ../nodejs/init-scripts:/docker-entrypoint-initdb.d
+
+  pg-storage:
+    image: postgres:latest
+    restart: always
+    environment:
+      - POSTGRES_USER=${PG_STORAGE_DATABASE_USER}
+      - POSTGRES_DB=${PG_STORAGE_DATABASE_NAME}
+      - POSTGRES_PASSWORD=${PG_STORAGE_DATABASE_PASSWORD}
+      - PGPORT=${PG_STORAGE_DATABASE_PORT}
+    volumes:
+      - pg_storage_data:/var/lib/postgresql/data
+    ports:
+      - "${PG_STORAGE_DATABASE_PORT}:${PG_STORAGE_DATABASE_PORT}"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${PG_STORAGE_DATABASE_USER} -d ${PG_STORAGE_DATABASE_NAME}"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
+volumes:
+  pg_storage_data:
+  pg_data:

demos/supabase/config/powersync.yaml

@@ -16,8 +16,9 @@ replication:
 
 # Connection settings for sync bucket storage
 storage:
-  type: mongodb
-  uri: !env PS_MONGO_URI
+  type: postgresql
+  uri: !env PS_STORAGE_SOURCE_URI
+  sslmode: disable # 'verify-full' (default) or 'verify-ca' or 'disable'
 
 # The port which the PowerSync API server will listen on
 port: !env PS_PORT

services/powersync.yaml

@@ -1,7 +1,9 @@
 services:
   powersync:
     restart: unless-stopped
-    image: journeyapps/powersync-service:latest
+    image:
+      # Postgres bucker storage is not yet available in the latest release
+      journeyapps/powersync-service:0.0.0-dev-20250113080324
     # The unified service runs an API server and replication worker in the same container.
     # These services can be executed in different containers by using individual entry commands e.g.
     # Start only the API server with