feat: use embedded postgres on Windows

.trunk/configs/cspell.json

@@ -44,13 +44,15 @@
     "dsname",
     "dspc",
     "dynamicmap",
+    "embeddedpostgres",
     "embedder",
     "embedders",
     "envfiles",
     "estree",
     "euclidian",
     "expfmt",
     "fatih",
+    "fergusstrange",
     "fkey",
     "fnptr",
     "fptr",
@@ -84,6 +86,7 @@
     "idof",
     "iface",
     "Infof",
+    "iofs",
     "isatty",
     "isize",
     "jackc",

.vscode/launch.json

@@ -10,8 +10,7 @@
       "env": {
         "FORCE_COLOR": "1",
         "MODUS_ENV": "dev",
-        "MODUS_DEBUG": "true",
-        "MODUS_DB": "postgresql://postgres:postgres@localhost:5433/my-runtime-db?sslmode=disable" // checkov:skip=CKV_SECRET_4
+        "MODUS_DEBUG": "true"
       },
       "args": [
         "--refresh=1s",
@@ -28,8 +27,7 @@
       "env": {
         "FORCE_COLOR": "1",
         "MODUS_ENV": "dev",
-        "MODUS_DEBUG": "true",
-        "MODUS_DB": "postgresql://postgres:postgres@localhost:5433/my-runtime-db?sslmode=disable" // checkov:skip=CKV_SECRET_4
+        "MODUS_DEBUG": "true"
       },
       "args": ["--refresh=1s", "--appPath", "${input:appPath}"]
     },
@@ -46,8 +44,7 @@
         "MODUS_DEBUG": "true",
         "AWS_REGION": "${input:awsRegion}",
         "AWS_PROFILE": "${input:awsProfile}",
-        "AWS_SDK_LOAD_CONFIG": "true",
-        "MODUS_DB": "postgresql://postgres:postgres@localhost:5433/my-runtime-db?sslmode=disable" // checkov:skip=CKV_SECRET_4
+        "AWS_SDK_LOAD_CONFIG": "true"
       },
       "args": [
         "--useAwsStorage",

CHANGELOG.md

@@ -5,6 +5,7 @@
 ## UNRELEASED
 
 - fix: omit parallel_tool_calls in Go OpenAI SDK if it is set to true [#849](https://github.com/hypermodeinc/modus/pull/849)
+- feat: use embedded postgres on Windows [#851](https://github.com/hypermodeinc/modus/pull/851)
 
 ## 2025-05-19 - Go SDK 0.18.0-alpha.2
 

runtime/db/db.go

@@ -51,6 +51,10 @@ func Stop(ctx context.Context) {
 	close(globalRuntimePostgresWriter.quit)
 	<-globalRuntimePostgresWriter.done
 	pool.Close()
+
+	if _embeddedPostgresDB != nil {
+		shutdownEmbeddedPostgresDB(ctx)
+	}
 }
 
 func logDbWarningOrError(ctx context.Context, err error, msg string) {
@@ -393,9 +397,20 @@ func QueryCollectionVectorsFromCheckpoint(ctx context.Context, collectionName, s
 }
 
 func Initialize(ctx context.Context) {
+	if useModusDB() {
+		return
+	}
+
+	if useEmbeddedPostgres() {
+		if err := prepareEmbeddedPostgresDB(ctx); err != nil {
+			logger.Fatal(ctx).Err(err).Msg("Failed to prepare embedded Postgres database.")
+			return
+		}
+	}
+
 	// this will initialize the pool and start the worker
 	_, err := globalRuntimePostgresWriter.GetPool(ctx)
-	if err != nil && !useModusDB() {
+	if err != nil {
 		logger.Warn(ctx).Err(err).Msg("Metadata database is not available.")
 	}
 	go globalRuntimePostgresWriter.worker(ctx)
@@ -437,7 +452,6 @@ var _useModusDB bool
 
 func useModusDB() bool {
 	_useModusDBOnce.Do(func() {
-		// this gives us a way to force the use or disuse of ModusDB for development
 		s := os.Getenv("MODUS_DB_USE_MODUSDB")
 		if s != "" {
 			if value, err := strconv.ParseBool(s); err == nil {
@@ -446,8 +460,11 @@ func useModusDB() bool {
 			}
 		}
 
-		// otherwise, it's based on the environment
-		_useModusDB = app.IsDevEnvironment()
+		if app.IsDevEnvironment() {
+			_useModusDB = !useEmbeddedPostgres()
+		} else {
+			_useModusDB = false
+		}
 	})
 	return _useModusDB
 }

runtime/db/embeddedpg.go

@@ -0,0 +1,226 @@
+/*
+ * Copyright 2025 Hypermode Inc.
+ * Licensed under the terms of the Apache License, Version 2.0
+ * See the LICENSE file that accompanied this code for further details.
+ *
+ * SPDX-FileCopyrightText: 2025 Hypermode Inc. <[email protected]>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package db
+
+import (
+	"bufio"
+	"context"
+	"embed"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+
+	"github.com/fatih/color"
+	"github.com/hypermodeinc/modus/runtime/app"
+	"github.com/hypermodeinc/modus/runtime/logger"
+	"github.com/rs/zerolog"
+
+	embeddedpostgres "github.com/fergusstrange/embedded-postgres"
+	"github.com/golang-migrate/migrate/v4"
+	_ "github.com/golang-migrate/migrate/v4/database/postgres"
+	"github.com/golang-migrate/migrate/v4/source/iofs"
+	_ "github.com/golang-migrate/migrate/v4/source/iofs"
+)
+
+//go:embed migrations/*.sql
+var migrationsFS embed.FS
+
+func useEmbeddedPostgres() bool {
+	s := os.Getenv("MODUS_DB_USE_EMBEDDED_POSTGRES")
+	if s != "" {
+		if value, err := strconv.ParseBool(s); err == nil {
+			return value
+		}
+	}
+	return runtime.GOOS == "windows"
+}
+
+var _embeddedPostgresDB *embeddedpostgres.EmbeddedPostgres
+
+func getEmbeddedPostgresDataDir(ctx context.Context) string {
+	var dataDir string
+	appPath := app.Config().AppPath()
+	if filepath.Base(appPath) == "build" {
+		// this keeps the data directory outside of the build directory
+		dataDir = filepath.Join(appPath, "..", ".postgres")
+		addToGitIgnore(ctx, filepath.Dir(appPath), ".postgres/")
+	} else {
+		dataDir = filepath.Join(appPath, ".postgres")
+	}
+	return dataDir
+}
+
+func prepareEmbeddedPostgresDB(ctx context.Context) error {
+
+	dataDir := getEmbeddedPostgresDataDir(ctx)
+
+	if err := shutdownPreviousEmbeddedPostgresDB(ctx, dataDir); err != nil {
+		return fmt.Errorf("error shutting down previous embedded postgres instance: %w", err)
+	}
+
+	port, err := findAvailablePort(5432, 5499)
+	if err != nil {
+		return err
+	}
+
+	logger.Info(ctx).Msg("Preparing embedded PostgreSQL database.  The db instance will log its output next:")
+
+	// Note: releases come from here:
+	// https://github.com/zonkyio/embedded-postgres-binaries/releases
+
+	dbname := "modusdb"
+	cfg := embeddedpostgres.DefaultConfig().
+		Port(port).
+		Database(dbname).
+		DataPath(dataDir).
+		Version("17.4.0").
+		OwnProcessGroup(true).
+		BinariesPath(getEmbeddedPostgresBinariesPath())
+
+	db := embeddedpostgres.NewDatabase(cfg)
+	if err := db.Start(); err != nil {
+		return fmt.Errorf("failed to start embedded postgres: %w", err)
+	}
+
+	cs := fmt.Sprintf("postgres://postgres:postgres@localhost:%d/%s?sslmode=disable", port, dbname)
+
+	d, err := iofs.New(migrationsFS, "migrations")
+	if err != nil {
+		return fmt.Errorf("error creating iofs source: %w", err)
+	}
+
+	m, err := migrate.NewWithSourceInstance("iofs", d, cs)
+	if err != nil {
+		return fmt.Errorf("error creating db migrate instance: %w", err)
+	}
+	m.Log = &migrateLogger{
+		logger: logger.Get(ctx),
+	}
+
+	if err := m.Up(); err != nil && err != migrate.ErrNoChange {
+		return fmt.Errorf("error running db migrations: %w", err)
+	}
+
+	_embeddedPostgresDB = db
+	os.Setenv("MODUS_DB", cs)
+
+	logger.Info(ctx).Msg("Embedded PostgreSQL database started successfully.")
+	return nil
+}
+
+func getEmbeddedPostgresBinariesPath() string {
+	return filepath.Join(app.ModusHomeDir(), "postgres")
+}
+
+func findAvailablePort(startPort, maxPort uint32) (uint32, error) {
+	for port := startPort; port <= maxPort; port++ {
+		addr := fmt.Sprintf("localhost:%d", port)
+		listener, err := net.Listen("tcp", addr)
+		if err == nil {
+			listener.Close()
+			return port, nil
+		}
+	}
+	return 0, fmt.Errorf("no available db ports between %d and %d", startPort, maxPort)
+}
+
+func shutdownEmbeddedPostgresDB(ctx context.Context) {
+	logger.Info(ctx).Msg("Shutting down embedded PostgreSQL database server:")
+
+	if err := _embeddedPostgresDB.Stop(); err != nil {
+		logger.Error(ctx).Err(err).Msg("Failed to stop embedded PostgreSQL database.")
+	}
+}
+
+func shutdownPreviousEmbeddedPostgresDB(ctx context.Context, dataDir string) error {
+
+	// does dataDir exist?
+	if _, err := os.Stat(dataDir); err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("error checking dataDir: %w", err)
+	}
+
+	// does `postmaster.pid` exist?
+	pidFile := filepath.Join(dataDir, "postmaster.pid")
+	if _, err := os.Stat(pidFile); err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+	}
+
+	logger.Warn(ctx).Msg("Previous embedded PostgreSQL instance was not shut down cleanly.  Shutting it down now:")
+
+	// first try to shutdown the process cleanly
+	pgctl := filepath.Join(getEmbeddedPostgresBinariesPath(), "bin", "pg_ctl")
+	if runtime.GOOS == "windows" {
+		pgctl += ".exe"
+	}
+	if _, err := os.Stat(pgctl); err == nil {
+		p := exec.Command(pgctl, "stop", "-w", "-D", dataDir)
+		p.Stdout = os.Stdout
+		p.Stderr = os.Stderr
+		if err := p.Run(); err == nil {
+			return nil
+		} else {
+			logger.Err(ctx, err).Msg("Failed to stop embedded PostgreSQL instance cleanly. Killing db process.")
+		}
+	}
+
+	// read the pid from the first line of the file
+	file, err := os.Open(pidFile)
+	if err != nil {
+		return fmt.Errorf("error opening pid file: %w", err)
+	}
+	defer file.Close()
+	scanner := bufio.NewScanner(file)
+	if scanner.Scan() {
+		pidStr := scanner.Text()
+		pid, err := strconv.Atoi(pidStr)
+		if err != nil {
+			return fmt.Errorf("error parsing pid: %w", err)
+		}
+
+		// kill the process
+		process, err := os.FindProcess(pid)
+		if err != nil {
+			return fmt.Errorf("error finding process: %w", err)
+		}
+		if err := process.Kill(); err != nil {
+			return fmt.Errorf("error killing process: %w", err)
+		}
+	}
+
+	return nil
+}
+
+var migrateLoggerColor = color.New(color.FgWhite, color.Faint)
+
+type migrateLogger struct {
+	logger  *zerolog.Logger
+	started bool
+}
+
+func (l *migrateLogger) Printf(format string, v ...interface{}) {
+	if !l.started {
+		l.logger.Info().Msg("Applying db migrations:")
+		l.started = true
+	}
+	migrateLoggerColor.Fprintf(os.Stderr, "  "+format, v...)
+}
+
+func (l *migrateLogger) Verbose() bool {
+	return false
+}