Improved Migrations (#628)

If running many migrations at once, do each in a separate transaction
instead of all in one transaction. Reduces contention when migrating an
active database.

Author: kraftp

dbos/_migration.py

@@ -45,23 +45,23 @@ def run_dbos_migrations(
     engine: sa.Engine, schema: str, use_listen_notify: bool
 ) -> None:
     """Run DBOS-managed migrations by executing each SQL command in dbos_migrations."""
+    # Get current migration version
     with engine.begin() as conn:
-        # Get current migration version
         result = conn.execute(
             sa.text(f'SELECT version FROM "{schema}".dbos_migrations')
         )
         current_version = result.fetchone()
         last_applied = current_version[0] if current_version else 0
 
-        # Apply migrations starting from the next version
-        migrations = get_dbos_migrations(schema, use_listen_notify)
-        for i, migration_sql in enumerate(migrations, 1):
-            if i <= last_applied:
-                continue
+    # Apply each migration in its own transaction
+    migrations = get_dbos_migrations(schema, use_listen_notify)
+    for i, migration_sql in enumerate(migrations, 1):
+        if i <= last_applied:
+            continue
 
-            # Execute the migration
-            dbos_logger.info(f"Applying DBOS system database schema migration {i}")
+        dbos_logger.info(f"Applying DBOS system database schema migration {i}")
 
+        with engine.begin() as conn:
             # Migration 10 adds a primary key to the notifications table.
             # Skip it if the table already has one.
             if (

dbos/cli/cli.py

@@ -24,7 +24,7 @@
     load_config,
 )
 from .._docker_pg_helper import start_docker_pg, stop_docker_pg
-from .._logger import dbos_logger
+from .._logger import dbos_logger, init_logger
 from .._sys_db import SystemDatabase
 from .._utils import GlobalParams
 from ..cli._github_init import create_template_from_github
@@ -286,7 +286,9 @@ def migrate(
         system_database_url=system_database_url,
         application_database_url=application_database_url,
     )
-
+    # Emit INFO logs from migrations
+    init_logger()
+    dbos_logger.setLevel(logging.INFO)
     typer.echo(f"Starting DBOS migrations")
     if application_database_url:
         typer.echo(f"Application database: {sa.make_url(application_database_url)}")