Local postgresql with custom dynamic database support for development+CI (#9022)

isoos · web-flow · commit 155c09a3fdde · 2025-10-20T16:41:30.000+02:00
diff --git a/.github/workflows/all-test.yml b/.github/workflows/all-test.yml
@@ -97,6 +97,19 @@ jobs:
     name: pkg/${{matrix.package}}/
     runs-on: ubuntu-latest
     needs: define_pkg_list
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
     strategy:
       fail-fast: false
       matrix:
@@ -137,3 +150,5 @@ jobs:
         fi
         dart test --run-skipped
       working-directory: pkg/${{matrix.package}}
+      env:
+        PUB_POSTGRES_URL: postgresql://postgres:postgres@localhost:5432/postgres?sslmode=disable
diff --git a/app/lib/database/database.dart b/app/lib/database/database.dart
@@ -2,12 +2,19 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:async';
+import 'dart:io';
+import 'dart:math';
+
+import 'package:clock/clock.dart';
 import 'package:gcloud/service_scope.dart' as ss;
 import 'package:meta/meta.dart';
 import 'package:postgres/postgres.dart';
 import 'package:pub_dev/service/secret/backend.dart';
 import 'package:pub_dev/shared/env_config.dart';
 
+final _random = Random.secure();
+
 /// Sets the primary database service.
 void registerPrimaryDatabase(PrimaryDatabase database) =>
     ss.register(#_primaryDatabase, database);
@@ -26,13 +33,38 @@ class PrimaryDatabase {
   /// the secret backend, connects to it and registers the primary database
   /// service in the current scope.
   static Future<void> tryRegisterInScope() async {
-    final connectionString =
+    var connectionString =
         envConfig.pubPostgresUrl ??
         (await secretBackend.lookup(SecretKey.postgresConnectionString));
-    if (connectionString == null) {
+    if (connectionString == null && envConfig.isRunningInAppengine) {
       // ignore for now, must throw once we have the environment setup ready
       return;
     }
+    // The scope-specific custom database. We are creating a custom database for
+    // each test run, in order to provide full isolation, however, this must not
+    // be used in Appengine.
+    String? customDb;
+    if (connectionString == null) {
+      (connectionString, customDb) = await _startOrUseLocalPostgresInDocker();
+    }
+    if (customDb == null && !envConfig.isRunningInAppengine) {
+      customDb = await _createCustomDatabase(connectionString);
+    }
+
+    if (customDb != null) {
+      if (envConfig.isRunningInAppengine) {
+        throw StateError('Should not use custom database inside AppEngine.');
+      }
+
+      final originalUrl = connectionString;
+      connectionString = Uri.parse(
+        connectionString,
+      ).replace(path: customDb).toString();
+      ss.registerScopeExitCallback(() async {
+        await _dropCustomDatabase(originalUrl, customDb!);
+      });
+    }
+
     final database = await _fromConnectionString(connectionString);
     registerPrimaryDatabase(database);
     ss.registerScopeExitCallback(database.close);
@@ -48,10 +80,64 @@ class PrimaryDatabase {
   }
 
   @visibleForTesting
-  Future<void> verifyConnection() async {
-    final rs = await _pg.execute('SELECT 1');
+  Future<String> verifyConnection() async {
+    final rs = await _pg.execute('SELECT current_database();');
     if (rs.length != 1) {
       throw StateError('Connection is not returning expected rows.');
     }
+    return rs.single.single as String;
+  }
+}
+
+Future<(String, String?)> _startOrUseLocalPostgresInDocker() async {
+  // sanity check
+  if (envConfig.isRunningInAppengine) {
+    throw StateError('Missing connection URL in Appengine environment.');
+  }
+
+  // the default connection URL for local server
+  final url = Uri(
+    scheme: 'postgresql',
+    host: 'localhost',
+    port: 55432,
+    path: 'postgres',
+    userInfo: 'postgres:postgres',
+    queryParameters: {'sslmode': 'disable'},
+  ).toString();
+
+  try {
+    // try opening the connection
+    final customDb = await _createCustomDatabase(url);
+    return (url, customDb);
+  } catch (_) {
+    // on failure start the local server
+    final pr = await Process.run('tool/start-local-postgres.sh', []);
+    if (pr.exitCode != 0) {
+      throw StateError(
+        'Unexpect exit code from tool/start-local-postgres.sh\n${pr.stderr}',
+      );
+    }
   }
+  return (url, null);
+}
+
+int _customDbCount = 0;
+
+Future<String> _createCustomDatabase(String url) async {
+  _customDbCount++;
+  final dbName =
+      'fake_pub_${pid.toRadixString(36)}'
+      '${_customDbCount.toRadixString(36)}'
+      '${clock.now().millisecondsSinceEpoch.toRadixString(36)}'
+      '${_random.nextInt(1 << 32).toRadixString(36)}';
+  final conn = await Connection.openFromUrl(url);
+  await conn.execute('CREATE DATABASE "$dbName";');
+  await conn.close(force: true);
+  return dbName;
+}
+
+Future<void> _dropCustomDatabase(String url, String dbName) async {
+  final conn = await Connection.openFromUrl(url);
+  await conn.execute('DROP DATABASE "$dbName";');
+  await conn.close(force: true);
 }
diff --git a/app/test/database/postgresql_ci_test.dart b/app/test/database/postgresql_ci_test.dart
@@ -39,12 +39,8 @@ void main() {
     testWithProfile(
       'registered database scope',
       fn: () async {
-        final pubPostgresUrl = envConfig.pubPostgresUrl;
-        if (pubPostgresUrl == null) {
-          markTestSkipped('PUB_POSTGRES_URL was not specified.');
-          return;
-        }
-        await primaryDatabase!.verifyConnection();
+        final name = await primaryDatabase!.verifyConnection();
+        expect(name, contains('fake_pub_'));
       },
     );
   });
diff --git a/app/tool/docker-postgres-timeout-entrypoint.sh b/app/tool/docker-postgres-timeout-entrypoint.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+set -e
+
+/usr/bin/timeout 6h /usr/local/bin/docker-entrypoint.sh $*
diff --git a/app/tool/start-local-postgres.sh b/app/tool/start-local-postgres.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+set -e
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+PG_TEMP_DIR="${TMPDIR:-/tmp}/pub_dev_postgres/"
+mkdir -p "${PG_TEMP_DIR}"
+
+# Create directory for exposing sockets
+SOCKET_DIR="${PG_TEMP_DIR}/run/"
+mkdir -p "$SOCKET_DIR"
+
+# Use an extra lock file to avoid every creating more than one docker container
+LOCKFILE="${PG_TEMP_DIR}/.docker.lock"
+touch "${LOCKFILE}"
+
+CONTAINER_ID=$((
+  flock -ox 200
+  if ! docker inspect 'pub_dev_postgres' > /dev/null 2>&1; then
+    docker run \
+      --detach \
+      --rm \
+      --name pub_dev_postgres \
+      -e POSTGRES_PASSWORD=postgres \
+      -v "${SCRIPT_DIR}/docker-postgres-timeout-entrypoint.sh":/pub-entrypoint.sh \
+      -v "${SOCKET_DIR}":/var/run/postgresql/ \
+      -p 55432:5432 \
+      --mount type=tmpfs,destination=/var/lib/postgresql/data \
+      --entrypoint /pub-entrypoint.sh \
+      postgres:17 \
+      postgres \
+        -c fsync=off \
+        -c synchronous_commit=off \
+        -c full_page_writes=off \
+        -c wal_level=minimal \
+        -c max_wal_senders=0 \
+        -c archive_mode=off
+  fi
+) 200>"$LOCKFILE")
+
+if [ -n "$CONTAINER_ID" ]; then
+  if [ "$1" != '--quiet' ]; then
+    echo 'Started postgres test database. Will auto-terminate in 6 hours.'
+  fi
+else
+  if [ "$1" != '--quiet' ]; then
+    echo 'Found postgres test database already running!'
+    echo 'If you want to restart it, you can use with:'
+    echo 'docker kill pub_dev_postgres'
+  fi
+fi
