From 5543f6830c398cd0ff771be6ae245c2f4cee9e40 Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Fri, 18 Jul 2025 12:33:14 +0530 Subject: [PATCH 01/21] Pending changes for alembic Signed-off-by: Madhav Kandukuri --- mcpgateway/alembic/env.py | 13 ------------- mcpgateway/bootstrap_db.py | 2 +- mcpgateway/main.py | 10 +--------- 3 files changed, 2 insertions(+), 23 deletions(-) diff --git a/mcpgateway/alembic/env.py b/mcpgateway/alembic/env.py index 58341db8..5fa99dd1 100644 --- a/mcpgateway/alembic/env.py +++ b/mcpgateway/alembic/env.py @@ -57,10 +57,6 @@ from mcpgateway.config import settings from mcpgateway.db import Base -# from mcpgateway.db import get_metadata -# target_metadata = get_metadata() - - # Create config object - this is the standard way in Alembic config = getattr(context, "config", None) or Config() @@ -111,18 +107,9 @@ def _inside_alembic() -> bool: disable_existing_loggers=False, ) -# First-Party -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel target_metadata = Base.metadata -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - config.set_main_option( "sqlalchemy.url", settings.database_url, diff --git a/mcpgateway/bootstrap_db.py b/mcpgateway/bootstrap_db.py index 7142c377..c88d666a 100644 --- a/mcpgateway/bootstrap_db.py +++ b/mcpgateway/bootstrap_db.py @@ -59,8 +59,8 @@ async def main() -> None: if "gateways" not in insp.get_table_names(): logger.info("Empty DB detected - creating baseline schema") - Base.metadata.create_all(bind=conn) command.stamp(cfg, "head") + Base.metadata.create_all(bind=conn) else: command.upgrade(cfg, "head") diff --git a/mcpgateway/main.py b/mcpgateway/main.py index 5e19d1f9..7b6ef1af 100644 --- a/mcpgateway/main.py +++ b/mcpgateway/main.py @@ -147,15 +147,6 @@ # Wait for database to be ready before creating tables wait_for_db_ready(max_tries=int(settings.db_max_retries), interval=int(settings.db_retry_interval_ms) / 1000, sync=True) # Converting ms to s -# Create database tables -try: - loop = asyncio.get_running_loop() -except RuntimeError: - asyncio.run(bootstrap_db()) -else: - loop.create_task(bootstrap_db()) - - # Initialize services tool_service = ToolService() resource_service = ResourceService() @@ -209,6 +200,7 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]: """ logger.info("Starting MCP Gateway services") try: + await bootstrap_db() await tool_service.initialize() await resource_service.initialize() await prompt_service.initialize() From c27ce283fc45b7722caf3f329240b9266a6156d8 Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Mon, 21 Jul 2025 14:46:39 +0000 Subject: [PATCH 02/21] use bootstrap_db for e2e test_db Signed-off-by: Madhav Kandukuri --- tests/e2e/test_admin_apis.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/e2e/test_admin_apis.py b/tests/e2e/test_admin_apis.py index ed60e8e9..db7b0fb3 100644 --- a/tests/e2e/test_admin_apis.py +++ b/tests/e2e/test_admin_apis.py @@ -52,6 +52,8 @@ # First-Party from mcpgateway.db import Base from mcpgateway.main import app, get_db +from mcpgateway.config import settings +from mcpgateway.bootstrap_db import main as bootstrap_db # pytest.skip("Temporarily disabling this suite", allow_module_level=True) @@ -67,7 +69,7 @@ # Fixtures # ------------------------- @pytest_asyncio.fixture -async def temp_db(): +async def temp_db(monkeypatch): """ Create a temporary SQLite database for testing. @@ -77,17 +79,19 @@ async def temp_db(): """ # Create temporary file for SQLite database db_fd, db_path = tempfile.mkstemp(suffix=".db") + sqlite_url = f"sqlite:///{db_path}" + + monkeypatch.setattr(settings, "database_url", sqlite_url) + + await bootstrap_db() # Ensure the database is bootstrapped # Create engine with SQLite engine = create_engine( - f"sqlite:///{db_path}", + sqlite_url, connect_args={"check_same_thread": False}, poolclass=StaticPool, ) - # Create all tables - Base.metadata.create_all(bind=engine) - # Create session factory TestSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) From d2e88919e5fcb52b32022bc6b69e71d68c6b84a2 Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Mon, 21 Jul 2025 14:59:16 +0000 Subject: [PATCH 03/21] logging tables Signed-off-by: Madhav Kandukuri --- tests/e2e/test_admin_apis.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/e2e/test_admin_apis.py b/tests/e2e/test_admin_apis.py index db7b0fb3..8cae301e 100644 --- a/tests/e2e/test_admin_apis.py +++ b/tests/e2e/test_admin_apis.py @@ -35,6 +35,7 @@ os.environ["MCPGATEWAY_UI_ENABLED"] = "true" # Standard +import logging import tempfile from typing import AsyncGenerator from unittest.mock import patch @@ -45,15 +46,17 @@ from httpx import AsyncClient import pytest import pytest_asyncio -from sqlalchemy import create_engine +from sqlalchemy import create_engine, inspect from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import StaticPool # First-Party +from mcpgateway.bootstrap_db import main as bootstrap_db +from mcpgateway.config import settings from mcpgateway.db import Base from mcpgateway.main import app, get_db -from mcpgateway.config import settings -from mcpgateway.bootstrap_db import main as bootstrap_db + +logger = logging.getLogger(__name__) # pytest.skip("Temporarily disabling this suite", allow_module_level=True) @@ -92,6 +95,10 @@ async def temp_db(monkeypatch): poolclass=StaticPool, ) + insp = inspect(engine) + table_list = insp.get_table_names() + logger.info(f"πŸŽ‰ Tables in temp DB: {table_list}") + # Create session factory TestSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) From ecce8d6445af9a28838a7fadbce9dc2f5eaad16f Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Mon, 21 Jul 2025 15:25:13 +0000 Subject: [PATCH 04/21] testing Signed-off-by: Madhav Kandukuri --- tests/e2e/test_admin_apis.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/e2e/test_admin_apis.py b/tests/e2e/test_admin_apis.py index 8cae301e..732f4825 100644 --- a/tests/e2e/test_admin_apis.py +++ b/tests/e2e/test_admin_apis.py @@ -95,10 +95,8 @@ async def temp_db(monkeypatch): poolclass=StaticPool, ) - insp = inspect(engine) - table_list = insp.get_table_names() - logger.info(f"πŸŽ‰ Tables in temp DB: {table_list}") - + Base.metadata.create_all(bind=engine) + # Create session factory TestSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) From 08999d2908e1d09ef72362da892d412c92fe5da5 Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Mon, 21 Jul 2025 15:42:02 +0000 Subject: [PATCH 05/21] testing Signed-off-by: Madhav Kandukuri --- tests/e2e/test_admin_apis.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/e2e/test_admin_apis.py b/tests/e2e/test_admin_apis.py index 732f4825..7430089a 100644 --- a/tests/e2e/test_admin_apis.py +++ b/tests/e2e/test_admin_apis.py @@ -72,7 +72,7 @@ # Fixtures # ------------------------- @pytest_asyncio.fixture -async def temp_db(monkeypatch): +async def temp_db(): """ Create a temporary SQLite database for testing. @@ -84,9 +84,9 @@ async def temp_db(monkeypatch): db_fd, db_path = tempfile.mkstemp(suffix=".db") sqlite_url = f"sqlite:///{db_path}" - monkeypatch.setattr(settings, "database_url", sqlite_url) + # monkeypatch.setattr(settings, "database_url", sqlite_url) - await bootstrap_db() # Ensure the database is bootstrapped + # await bootstrap_db() # Ensure the database is bootstrapped # Create engine with SQLite engine = create_engine( @@ -95,8 +95,8 @@ async def temp_db(monkeypatch): poolclass=StaticPool, ) - Base.metadata.create_all(bind=engine) - + Base.metadata.create_all(bind=engine) # Create tables + # Create session factory TestSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) From 44f1715de8467dbe256c4aa9e918aad0fb9404e3 Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Mon, 21 Jul 2025 17:10:29 +0000 Subject: [PATCH 06/21] testing Signed-off-by: Madhav Kandukuri --- .../versions/9c07b4bc5774_auto_migration.py | 218 +++++++ .../b77ca9d2de7e_uuid_pk_and_slug_refactor.py | 552 ------------------ .../e4fc04d1a442_add_annotations_to_tables.py | 56 -- ...490e949b1_add_improved_status_to_tables.py | 44 -- mcpgateway/bootstrap_db.py | 14 +- tests/e2e/test_admin_apis.py | 9 +- 6 files changed, 231 insertions(+), 662 deletions(-) create mode 100644 mcpgateway/alembic/versions/9c07b4bc5774_auto_migration.py delete mode 100644 mcpgateway/alembic/versions/b77ca9d2de7e_uuid_pk_and_slug_refactor.py delete mode 100644 mcpgateway/alembic/versions/e4fc04d1a442_add_annotations_to_tables.py delete mode 100644 mcpgateway/alembic/versions/e75490e949b1_add_improved_status_to_tables.py diff --git a/mcpgateway/alembic/versions/9c07b4bc5774_auto_migration.py b/mcpgateway/alembic/versions/9c07b4bc5774_auto_migration.py new file mode 100644 index 00000000..8cfc5c59 --- /dev/null +++ b/mcpgateway/alembic/versions/9c07b4bc5774_auto_migration.py @@ -0,0 +1,218 @@ +"""auto migration + +Revision ID: 9c07b4bc5774 +Revises: +Create Date: 2025-07-21 17:05:00.624436 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '9c07b4bc5774' +down_revision: Union[str, Sequence[str], None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('gateways', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('slug', sa.String(), nullable=False), + sa.Column('url', sa.String(), nullable=False), + sa.Column('description', sa.String(), nullable=True), + sa.Column('transport', sa.String(), nullable=False), + sa.Column('capabilities', sa.JSON(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('enabled', sa.Boolean(), nullable=False), + sa.Column('reachable', sa.Boolean(), nullable=False), + sa.Column('last_seen', sa.DateTime(), nullable=True), + sa.Column('auth_type', sa.String(), nullable=True), + sa.Column('auth_value', sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('slug'), + sa.UniqueConstraint('url') + ) + op.create_table('mcp_sessions', + sa.Column('session_id', sa.String(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('last_accessed', sa.DateTime(timezone=True), nullable=False), + sa.Column('data', sa.String(), nullable=True), + sa.PrimaryKeyConstraint('session_id') + ) + op.create_table('servers', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('description', sa.String(), nullable=True), + sa.Column('icon', sa.String(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('is_active', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + op.create_table('mcp_messages', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('session_id', sa.String(), nullable=False), + sa.Column('message', sa.String(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('last_accessed', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['session_id'], ['mcp_sessions.session_id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('prompts', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('description', sa.String(), nullable=True), + sa.Column('template', sa.Text(), nullable=False), + sa.Column('argument_schema', sa.JSON(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('is_active', sa.Boolean(), nullable=False), + sa.Column('gateway_id', sa.String(length=36), nullable=True), + sa.ForeignKeyConstraint(['gateway_id'], ['gateways.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + op.create_table('resources', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uri', sa.String(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('description', sa.String(), nullable=True), + sa.Column('mime_type', sa.String(), nullable=True), + sa.Column('size', sa.Integer(), nullable=True), + sa.Column('template', sa.String(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('is_active', sa.Boolean(), nullable=False), + sa.Column('text_content', sa.Text(), nullable=True), + sa.Column('binary_content', sa.LargeBinary(), nullable=True), + sa.Column('gateway_id', sa.String(length=36), nullable=True), + sa.ForeignKeyConstraint(['gateway_id'], ['gateways.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('uri') + ) + op.create_table('server_metrics', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('server_id', sa.String(), nullable=False), + sa.Column('timestamp', sa.DateTime(timezone=True), nullable=False), + sa.Column('response_time', sa.Float(), nullable=False), + sa.Column('is_success', sa.Boolean(), nullable=False), + sa.Column('error_message', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['server_id'], ['servers.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('tools', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('original_name', sa.String(), nullable=False), + sa.Column('original_name_slug', sa.String(), nullable=False), + sa.Column('url', sa.String(), nullable=True), + sa.Column('description', sa.String(), nullable=True), + sa.Column('integration_type', sa.String(), nullable=False), + sa.Column('request_type', sa.String(), nullable=False), + sa.Column('headers', sa.JSON(), nullable=True), + sa.Column('input_schema', sa.JSON(), nullable=False), + sa.Column('annotations', sa.JSON(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('enabled', sa.Boolean(), nullable=False), + sa.Column('reachable', sa.Boolean(), nullable=False), + sa.Column('jsonpath_filter', sa.String(), nullable=False), + sa.Column('auth_type', sa.String(), nullable=True), + sa.Column('auth_value', sa.String(), nullable=True), + sa.Column('gateway_id', sa.String(length=36), nullable=True), + sa.Column('name', sa.String(), nullable=True), + sa.ForeignKeyConstraint(['gateway_id'], ['gateways.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('gateway_id', 'original_name', name='uq_gateway_id__original_name'), + sa.UniqueConstraint('name') + ) + op.create_table('prompt_metrics', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('prompt_id', sa.Integer(), nullable=False), + sa.Column('timestamp', sa.DateTime(timezone=True), nullable=False), + sa.Column('response_time', sa.Float(), nullable=False), + sa.Column('is_success', sa.Boolean(), nullable=False), + sa.Column('error_message', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['prompt_id'], ['prompts.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('resource_metrics', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('resource_id', sa.Integer(), nullable=False), + sa.Column('timestamp', sa.DateTime(timezone=True), nullable=False), + sa.Column('response_time', sa.Float(), nullable=False), + sa.Column('is_success', sa.Boolean(), nullable=False), + sa.Column('error_message', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['resource_id'], ['resources.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('resource_subscriptions', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('resource_id', sa.Integer(), nullable=False), + sa.Column('subscriber_id', sa.String(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('last_notification', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['resource_id'], ['resources.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('server_prompt_association', + sa.Column('server_id', sa.String(), nullable=False), + sa.Column('prompt_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['prompt_id'], ['prompts.id'], ), + sa.ForeignKeyConstraint(['server_id'], ['servers.id'], ), + sa.PrimaryKeyConstraint('server_id', 'prompt_id') + ) + op.create_table('server_resource_association', + sa.Column('server_id', sa.String(), nullable=False), + sa.Column('resource_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['resource_id'], ['resources.id'], ), + sa.ForeignKeyConstraint(['server_id'], ['servers.id'], ), + sa.PrimaryKeyConstraint('server_id', 'resource_id') + ) + op.create_table('server_tool_association', + sa.Column('server_id', sa.String(), nullable=False), + sa.Column('tool_id', sa.String(), nullable=False), + sa.ForeignKeyConstraint(['server_id'], ['servers.id'], ), + sa.ForeignKeyConstraint(['tool_id'], ['tools.id'], ), + sa.PrimaryKeyConstraint('server_id', 'tool_id') + ) + op.create_table('tool_metrics', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('tool_id', sa.String(), nullable=False), + sa.Column('timestamp', sa.DateTime(timezone=True), nullable=False), + sa.Column('response_time', sa.Float(), nullable=False), + sa.Column('is_success', sa.Boolean(), nullable=False), + sa.Column('error_message', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['tool_id'], ['tools.id'], ), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('tool_metrics') + op.drop_table('server_tool_association') + op.drop_table('server_resource_association') + op.drop_table('server_prompt_association') + op.drop_table('resource_subscriptions') + op.drop_table('resource_metrics') + op.drop_table('prompt_metrics') + op.drop_table('tools') + op.drop_table('server_metrics') + op.drop_table('resources') + op.drop_table('prompts') + op.drop_table('mcp_messages') + op.drop_table('servers') + op.drop_table('mcp_sessions') + op.drop_table('gateways') + # ### end Alembic commands ### diff --git a/mcpgateway/alembic/versions/b77ca9d2de7e_uuid_pk_and_slug_refactor.py b/mcpgateway/alembic/versions/b77ca9d2de7e_uuid_pk_and_slug_refactor.py deleted file mode 100644 index 0170e4ac..00000000 --- a/mcpgateway/alembic/versions/b77ca9d2de7e_uuid_pk_and_slug_refactor.py +++ /dev/null @@ -1,552 +0,0 @@ -# -*- coding: utf-8 -*- -"""uuid-pk_and_slug_refactor - -Revision ID: b77ca9d2de7e -Revises: -Create Date: 2025-06-26 21:29:59.117140 - -""" - -# Standard -from typing import Sequence, Union -import uuid - -# Third-Party -from alembic import op -import sqlalchemy as sa -from sqlalchemy.orm import Session - -# First-Party -from mcpgateway.config import settings -from mcpgateway.utils.create_slug import slugify - -# revision identifiers, used by Alembic. -revision: str = "b77ca9d2de7e" -down_revision: Union[str, Sequence[str], None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -# ────────────────────────────────────────────────────────────────────────────── -# Helpers -# ────────────────────────────────────────────────────────────────────────────── -def _use_batch() -> bool: - """Determine if batch operations are required for the current database. - - SQLite requires batch mode for certain ALTER TABLE operations like dropping - columns or altering column types. This helper checks the database dialect - to determine if batch operations should be used. - - Returns: - bool: True if the database is SQLite (requires batch mode), False otherwise. - - Examples: - >>> # In a SQLite context - >>> _use_batch() # doctest: +SKIP - True - >>> # In a PostgreSQL context - >>> _use_batch() # doctest: +SKIP - False - """ - return op.get_bind().dialect.name == "sqlite" - - -# ────────────────────────────────────────────────────────────────────────────── -# Upgrade -# ────────────────────────────────────────────────────────────────────────────── -def upgrade() -> None: - """Migrate database schema from integer to UUID primary keys with slugs. - - This migration performs a comprehensive schema transformation in three stages: - - Stage 1 - Add placeholder columns: - - Adds UUID columns (id_new) to gateways, tools, and servers - - Adds slug columns for human-readable identifiers - - Adds columns to preserve original tool names before prefixing - - Stage 2 - Data migration: - - Generates UUIDs for all primary keys - - Creates slugs from names (e.g., "My Gateway" -> "my-gateway") - - Prefixes tool names with gateway slugs (e.g., "my-tool" -> "gateway-slug-my-tool") - - Updates all foreign key references to use new UUIDs - - Stage 3 - Schema finalization: - - Drops old integer columns - - Renames new UUID columns to replace old ones - - Recreates primary keys and foreign key constraints - - Adds unique constraints on slugs and URLs - - The migration is designed to work with both SQLite (using batch operations) - and other databases. It preserves all existing data relationships while - transforming the schema. - - Note: - - Skips migration if database is fresh (no gateways table) - - Uses batch operations for SQLite compatibility - - Commits data changes before schema alterations - - Examples: - >>> # Running the migration - >>> upgrade() # doctest: +SKIP - Fresh database detected. Skipping migration. - >>> # Or for existing database - >>> upgrade() # doctest: +SKIP - Existing installation detected. Starting data and schema migration... - """ - bind = op.get_bind() - sess = Session(bind=bind) - inspector = sa.inspect(bind) - - if not inspector.has_table("gateways"): - print("Fresh database detected. Skipping migration.") - return - - print("Existing installation detected. Starting data and schema migration...") - - # ── STAGE 1: ADD NEW NULLABLE COLUMNS AS PLACEHOLDERS ───────────────── - op.add_column("gateways", sa.Column("slug", sa.String(), nullable=True)) - op.add_column("gateways", sa.Column("id_new", sa.String(36), nullable=True)) - - op.add_column("tools", sa.Column("id_new", sa.String(36), nullable=True)) - op.add_column("tools", sa.Column("original_name", sa.String(), nullable=True)) - op.add_column("tools", sa.Column("original_name_slug", sa.String(), nullable=True)) - op.add_column("tools", sa.Column("name_new", sa.String(), nullable=True)) - op.add_column("tools", sa.Column("gateway_id_new", sa.String(36), nullable=True)) - - op.add_column("resources", sa.Column("gateway_id_new", sa.String(36), nullable=True)) - op.add_column("prompts", sa.Column("gateway_id_new", sa.String(36), nullable=True)) - - op.add_column("servers", sa.Column("id_new", sa.String(36), nullable=True)) - - op.add_column("server_tool_association", sa.Column("server_id_new", sa.String(36), nullable=True)) - op.add_column("server_tool_association", sa.Column("tool_id_new", sa.String(36), nullable=True)) - - op.add_column("tool_metrics", sa.Column("tool_id_new", sa.String(36), nullable=True)) - op.add_column("server_metrics", sa.Column("server_id_new", sa.String(36), nullable=True)) - op.add_column("server_resource_association", sa.Column("server_id_new", sa.String(36), nullable=True)) - op.add_column("server_prompt_association", sa.Column("server_id_new", sa.String(36), nullable=True)) - - # ── STAGE 2: POPULATE THE NEW COLUMNS (DATA MIGRATION) ─────────────── - gateways = sess.execute(sa.select(sa.text("id, name")).select_from(sa.text("gateways"))).all() - for gid, gname in gateways: - g_uuid = uuid.uuid4().hex - sess.execute( - sa.text("UPDATE gateways SET id_new=:u, slug=:s WHERE id=:i"), - {"u": g_uuid, "s": slugify(gname), "i": gid}, - ) - - tools = sess.execute(sa.select(sa.text("id, name, gateway_id")).select_from(sa.text("tools"))).all() - for tid, tname, g_old in tools: - t_uuid = uuid.uuid4().hex - tool_slug = slugify(tname) - sess.execute( - sa.text( - """ - UPDATE tools - SET id_new=:u, - original_name=:on, - original_name_slug=:ons, - name_new = CASE - WHEN :g IS NOT NULL THEN (SELECT slug FROM gateways WHERE id = :g) || :sep || :ons - ELSE :ons - END, - gateway_id_new=(SELECT id_new FROM gateways WHERE id=:g) - WHERE id=:i - """ - ), - { - "u": t_uuid, - "on": tname, - "ons": tool_slug, - "sep": settings.gateway_tool_name_separator, - "g": g_old, - "i": tid, - }, - ) - - servers = sess.execute(sa.select(sa.text("id")).select_from(sa.text("servers"))).all() - for (sid,) in servers: - sess.execute( - sa.text("UPDATE servers SET id_new=:u WHERE id=:i"), - {"u": uuid.uuid4().hex, "i": sid}, - ) - - # Populate all dependent tables - resources = sess.execute(sa.select(sa.text("id, gateway_id")).select_from(sa.text("resources"))).all() - for rid, g_old in resources: - sess.execute(sa.text("UPDATE resources SET gateway_id_new=(SELECT id_new FROM gateways WHERE id=:g) WHERE id=:i"), {"g": g_old, "i": rid}) - prompts = sess.execute(sa.select(sa.text("id, gateway_id")).select_from(sa.text("prompts"))).all() - for pid, g_old in prompts: - sess.execute(sa.text("UPDATE prompts SET gateway_id_new=(SELECT id_new FROM gateways WHERE id=:g) WHERE id=:i"), {"g": g_old, "i": pid}) - sta = sess.execute(sa.select(sa.text("server_id, tool_id")).select_from(sa.text("server_tool_association"))).all() - for s_old, t_old in sta: - sess.execute( - sa.text("UPDATE server_tool_association SET server_id_new=(SELECT id_new FROM servers WHERE id=:s), tool_id_new=(SELECT id_new FROM tools WHERE id=:t) WHERE server_id=:s AND tool_id=:t"), - {"s": s_old, "t": t_old}, - ) - tool_metrics = sess.execute(sa.select(sa.text("id, tool_id")).select_from(sa.text("tool_metrics"))).all() - for tmid, t_old in tool_metrics: - sess.execute(sa.text("UPDATE tool_metrics SET tool_id_new=(SELECT id_new FROM tools WHERE id=:t) WHERE id=:i"), {"t": t_old, "i": tmid}) - server_metrics = sess.execute(sa.select(sa.text("id, server_id")).select_from(sa.text("server_metrics"))).all() - for smid, s_old in server_metrics: - sess.execute(sa.text("UPDATE server_metrics SET server_id_new=(SELECT id_new FROM servers WHERE id=:s) WHERE id=:i"), {"s": s_old, "i": smid}) - server_resource_assoc = sess.execute(sa.select(sa.text("server_id, resource_id")).select_from(sa.text("server_resource_association"))).all() - for s_old, r_id in server_resource_assoc: - sess.execute(sa.text("UPDATE server_resource_association SET server_id_new=(SELECT id_new FROM servers WHERE id=:s) WHERE server_id=:s AND resource_id=:r"), {"s": s_old, "r": r_id}) - server_prompt_assoc = sess.execute(sa.select(sa.text("server_id, prompt_id")).select_from(sa.text("server_prompt_association"))).all() - for s_old, p_id in server_prompt_assoc: - sess.execute(sa.text("UPDATE server_prompt_association SET server_id_new=(SELECT id_new FROM servers WHERE id=:s) WHERE server_id=:s AND prompt_id=:p"), {"s": s_old, "p": p_id}) - - sess.commit() - - # ── STAGE 3: FINALIZE SCHEMA (CORRECTED ORDER) ─────────────────────── - # First, rebuild all tables that depend on `servers` and `gateways`. - # This implicitly drops their old foreign key constraints. - with op.batch_alter_table("server_tool_association") as batch_op: - batch_op.drop_column("server_id") - batch_op.drop_column("tool_id") - batch_op.alter_column("server_id_new", new_column_name="server_id", nullable=False) - batch_op.alter_column("tool_id_new", new_column_name="tool_id", nullable=False) - batch_op.create_primary_key("pk_server_tool_association", ["server_id", "tool_id"]) - - with op.batch_alter_table("server_resource_association") as batch_op: - batch_op.drop_column("server_id") - batch_op.alter_column("server_id_new", new_column_name="server_id", nullable=False) - - with op.batch_alter_table("server_prompt_association") as batch_op: - batch_op.drop_column("server_id") - batch_op.alter_column("server_id_new", new_column_name="server_id", nullable=False) - - with op.batch_alter_table("server_metrics") as batch_op: - batch_op.drop_column("server_id") - batch_op.alter_column("server_id_new", new_column_name="server_id", nullable=False) - - with op.batch_alter_table("tool_metrics") as batch_op: - batch_op.drop_column("tool_id") - batch_op.alter_column("tool_id_new", new_column_name="tool_id", nullable=False) - - with op.batch_alter_table("tools") as batch_op: - batch_op.drop_column("id") - batch_op.alter_column("id_new", new_column_name="id", nullable=False) - batch_op.create_primary_key("pk_tools", ["id"]) - batch_op.drop_column("gateway_id") - batch_op.alter_column("gateway_id_new", new_column_name="gateway_id", nullable=True) - batch_op.drop_column("name") - batch_op.alter_column("name_new", new_column_name="name", nullable=True) - batch_op.alter_column("original_name", nullable=False) - batch_op.alter_column("original_name_slug", nullable=False) - batch_op.create_unique_constraint("uq_tools_name", ["name"]) - batch_op.create_unique_constraint("uq_gateway_id__original_name", ["gateway_id", "original_name"]) - - with op.batch_alter_table("resources") as batch_op: - batch_op.drop_column("gateway_id") - batch_op.alter_column("gateway_id_new", new_column_name="gateway_id", nullable=True) - - with op.batch_alter_table("prompts") as batch_op: - batch_op.drop_column("gateway_id") - batch_op.alter_column("gateway_id_new", new_column_name="gateway_id", nullable=True) - - # Second, now that no tables point to their old IDs, rebuild `gateways` and `servers`. - with op.batch_alter_table("gateways") as batch_op: - batch_op.drop_column("id") - batch_op.alter_column("id_new", new_column_name="id", nullable=False) - batch_op.create_primary_key("pk_gateways", ["id"]) - batch_op.alter_column("slug", nullable=False) - batch_op.create_unique_constraint("uq_gateways_slug", ["slug"]) - batch_op.create_unique_constraint("uq_gateways_url", ["url"]) - - with op.batch_alter_table("servers") as batch_op: - batch_op.drop_column("id") - batch_op.alter_column("id_new", new_column_name="id", nullable=False) - batch_op.create_primary_key("pk_servers", ["id"]) - - # Finally, recreate all the foreign key constraints in batch mode for SQLite compatibility. - # The redundant `source_table` argument has been removed from each call. - with op.batch_alter_table("tools") as batch_op: - batch_op.create_foreign_key("fk_tools_gateway_id", "gateways", ["gateway_id"], ["id"]) - with op.batch_alter_table("resources") as batch_op: - batch_op.create_foreign_key("fk_resources_gateway_id", "gateways", ["gateway_id"], ["id"]) - with op.batch_alter_table("prompts") as batch_op: - batch_op.create_foreign_key("fk_prompts_gateway_id", "gateways", ["gateway_id"], ["id"]) - with op.batch_alter_table("server_tool_association") as batch_op: - batch_op.create_foreign_key("fk_server_tool_association_servers", "servers", ["server_id"], ["id"]) - batch_op.create_foreign_key("fk_server_tool_association_tools", "tools", ["tool_id"], ["id"]) - with op.batch_alter_table("tool_metrics") as batch_op: - batch_op.create_foreign_key("fk_tool_metrics_tool_id", "tools", ["tool_id"], ["id"]) - with op.batch_alter_table("server_metrics") as batch_op: - batch_op.create_foreign_key("fk_server_metrics_server_id", "servers", ["server_id"], ["id"]) - with op.batch_alter_table("server_resource_association") as batch_op: - batch_op.create_foreign_key("fk_server_resource_association_server_id", "servers", ["server_id"], ["id"]) - with op.batch_alter_table("server_prompt_association") as batch_op: - batch_op.create_foreign_key("fk_server_prompt_association_server_id", "servers", ["server_id"], ["id"]) - - -# def upgrade() -> None: -# bind = op.get_bind() -# sess = Session(bind=bind) -# inspector = sa.inspect(bind) - -# if not inspector.has_table("gateways"): -# print("Fresh database detected. Skipping migration.") -# return - -# print("Existing installation detected. Starting data and schema migration...") - -# # ── STAGE 1: ADD NEW NULLABLE COLUMNS AS PLACEHOLDERS ───────────────── -# op.add_column("gateways", sa.Column("slug", sa.String(), nullable=True)) -# op.add_column("gateways", sa.Column("id_new", sa.String(36), nullable=True)) - -# op.add_column("tools", sa.Column("id_new", sa.String(36), nullable=True)) -# op.add_column("tools", sa.Column("original_name", sa.String(), nullable=True)) -# op.add_column("tools", sa.Column("original_name_slug", sa.String(), nullable=True)) -# op.add_column("tools", sa.Column("name_new", sa.String(), nullable=True)) -# op.add_column("tools", sa.Column("gateway_id_new", sa.String(36), nullable=True)) - -# op.add_column("resources", sa.Column("gateway_id_new", sa.String(36), nullable=True)) -# op.add_column("prompts", sa.Column("gateway_id_new", sa.String(36), nullable=True)) - -# op.add_column("servers", sa.Column("id_new", sa.String(36), nullable=True)) - -# op.add_column("server_tool_association", sa.Column("server_id_new", sa.String(36), nullable=True)) -# op.add_column("server_tool_association", sa.Column("tool_id_new", sa.String(36), nullable=True)) - -# op.add_column("tool_metrics", sa.Column("tool_id_new", sa.String(36), nullable=True)) - -# # Add columns for the new server dependencies -# op.add_column("server_metrics", sa.Column("server_id_new", sa.String(36), nullable=True)) -# op.add_column("server_resource_association", sa.Column("server_id_new", sa.String(36), nullable=True)) -# op.add_column("server_prompt_association", sa.Column("server_id_new", sa.String(36), nullable=True)) - - -# # ── STAGE 2: POPULATE THE NEW COLUMNS (DATA MIGRATION) ─────────────── -# gateways = sess.execute(sa.select(sa.text("id, name")).select_from(sa.text("gateways"))).all() -# for gid, gname in gateways: -# g_uuid = uuid.uuid4().hex -# sess.execute( -# sa.text("UPDATE gateways SET id_new=:u, slug=:s WHERE id=:i"), -# {"u": g_uuid, "s": slugify(gname), "i": gid}, -# ) - -# tools = sess.execute( -# sa.select(sa.text("id, name, gateway_id")).select_from(sa.text("tools")) -# ).all() -# for tid, tname, g_old in tools: -# t_uuid = uuid.uuid4().hex -# tool_slug = slugify(tname) -# sess.execute( -# sa.text( -# """ -# UPDATE tools -# SET id_new=:u, -# original_name=:on, -# original_name_slug=:ons, -# name_new = CASE -# WHEN :g IS NOT NULL THEN (SELECT slug FROM gateways WHERE id = :g) || :sep || :ons -# ELSE :ons -# END, -# gateway_id_new=(SELECT id_new FROM gateways WHERE id=:g) -# WHERE id=:i -# """ -# ), -# { -# "u": t_uuid, "on": tname, "ons": tool_slug, -# "sep": settings.gateway_tool_name_separator, "g": g_old, "i": tid, -# }, -# ) - -# servers = sess.execute(sa.select(sa.text("id")).select_from(sa.text("servers"))).all() -# for (sid,) in servers: -# sess.execute( -# sa.text("UPDATE servers SET id_new=:u WHERE id=:i"), -# {"u": uuid.uuid4().hex, "i": sid}, -# ) - -# # Populate all dependent tables -# resources = sess.execute(sa.select(sa.text("id, gateway_id")).select_from(sa.text("resources"))).all() -# for rid, g_old in resources: -# sess.execute(sa.text("UPDATE resources SET gateway_id_new=(SELECT id_new FROM gateways WHERE id=:g) WHERE id=:i"), {"g": g_old, "i": rid}) -# prompts = sess.execute(sa.select(sa.text("id, gateway_id")).select_from(sa.text("prompts"))).all() -# for pid, g_old in prompts: -# sess.execute(sa.text("UPDATE prompts SET gateway_id_new=(SELECT id_new FROM gateways WHERE id=:g) WHERE id=:i"), {"g": g_old, "i": pid}) -# sta = sess.execute(sa.select(sa.text("server_id, tool_id")).select_from(sa.text("server_tool_association"))).all() -# for s_old, t_old in sta: -# sess.execute(sa.text("UPDATE server_tool_association SET server_id_new=(SELECT id_new FROM servers WHERE id=:s), tool_id_new=(SELECT id_new FROM tools WHERE id=:t) WHERE server_id=:s AND tool_id=:t"), {"s": s_old, "t": t_old}) -# tool_metrics = sess.execute(sa.select(sa.text("id, tool_id")).select_from(sa.text("tool_metrics"))).all() -# for tmid, t_old in tool_metrics: -# sess.execute(sa.text("UPDATE tool_metrics SET tool_id_new=(SELECT id_new FROM tools WHERE id=:t) WHERE id=:i"), {"t": t_old, "i": tmid}) -# server_metrics = sess.execute(sa.select(sa.text("id, server_id")).select_from(sa.text("server_metrics"))).all() -# for smid, s_old in server_metrics: -# sess.execute(sa.text("UPDATE server_metrics SET server_id_new=(SELECT id_new FROM servers WHERE id=:s) WHERE id=:i"), {"s": s_old, "i": smid}) -# server_resource_assoc = sess.execute(sa.select(sa.text("server_id, resource_id")).select_from(sa.text("server_resource_association"))).all() -# for s_old, r_id in server_resource_assoc: -# sess.execute(sa.text("UPDATE server_resource_association SET server_id_new=(SELECT id_new FROM servers WHERE id=:s) WHERE server_id=:s AND resource_id=:r"), {"s": s_old, "r": r_id}) -# server_prompt_assoc = sess.execute(sa.select(sa.text("server_id, prompt_id")).select_from(sa.text("server_prompt_association"))).all() -# for s_old, p_id in server_prompt_assoc: -# sess.execute(sa.text("UPDATE server_prompt_association SET server_id_new=(SELECT id_new FROM servers WHERE id=:s) WHERE server_id=:s AND prompt_id=:p"), {"s": s_old, "p": p_id}) - -# sess.commit() - -# # ── STAGE 3: FINALIZE SCHEMA (CORRECTED ORDER) ─────────────────────── -# with op.batch_alter_table("server_tool_association") as batch_op: -# batch_op.drop_column("server_id") -# batch_op.drop_column("tool_id") -# batch_op.alter_column("server_id_new", new_column_name="server_id", nullable=False) -# batch_op.alter_column("tool_id_new", new_column_name="tool_id", nullable=False) -# batch_op.create_primary_key("pk_server_tool_association", ["server_id", "tool_id"]) - -# with op.batch_alter_table("server_resource_association") as batch_op: -# batch_op.drop_column("server_id") -# batch_op.alter_column("server_id_new", new_column_name="server_id", nullable=False) - -# with op.batch_alter_table("server_prompt_association") as batch_op: -# batch_op.drop_column("server_id") -# batch_op.alter_column("server_id_new", new_column_name="server_id", nullable=False) - -# with op.batch_alter_table("server_metrics") as batch_op: -# batch_op.drop_column("server_id") -# batch_op.alter_column("server_id_new", new_column_name="server_id", nullable=False) - -# with op.batch_alter_table("tool_metrics") as batch_op: -# batch_op.drop_column("tool_id") -# batch_op.alter_column("tool_id_new", new_column_name="tool_id", nullable=False) - -# with op.batch_alter_table("tools") as batch_op: -# batch_op.drop_column("id") -# batch_op.alter_column("id_new", new_column_name="id", nullable=False) -# batch_op.create_primary_key("pk_tools", ["id"]) -# batch_op.drop_column("gateway_id") -# batch_op.alter_column("gateway_id_new", new_column_name="gateway_id", nullable=True) -# batch_op.drop_column("name") -# batch_op.alter_column("name_new", new_column_name="name", nullable=False) -# batch_op.alter_column("original_name", nullable=False) -# batch_op.alter_column("original_name_slug", nullable=False) -# batch_op.create_unique_constraint("uq_tools_name", ["name"]) -# batch_op.create_unique_constraint("uq_gateway_id__original_name", ["gateway_id", "original_name"]) - -# with op.batch_alter_table("resources") as batch_op: -# batch_op.drop_column("gateway_id") -# batch_op.alter_column("gateway_id_new", new_column_name="gateway_id", nullable=True) - -# with op.batch_alter_table("prompts") as batch_op: -# batch_op.drop_column("gateway_id") -# batch_op.alter_column("gateway_id_new", new_column_name="gateway_id", nullable=True) - -# with op.batch_alter_table("gateways") as batch_op: -# batch_op.drop_column("id") -# batch_op.alter_column("id_new", new_column_name="id", nullable=False) -# batch_op.create_primary_key("pk_gateways", ["id"]) -# batch_op.alter_column("slug", nullable=False) -# batch_op.create_unique_constraint("uq_gateways_slug", ["slug"]) -# batch_op.create_unique_constraint("uq_gateways_url", ["url"]) - -# with op.batch_alter_table("servers") as batch_op: -# batch_op.drop_column("id") -# batch_op.alter_column("id_new", new_column_name="id", nullable=False) -# batch_op.create_primary_key("pk_servers", ["id"]) - -# # Finally, recreate all the foreign key constraints -# op.create_foreign_key("fk_tools_gateway_id", "tools", "gateways", ["gateway_id"], ["id"]) -# op.create_foreign_key("fk_resources_gateway_id", "resources", "gateways", ["gateway_id"], ["id"]) -# op.create_foreign_key("fk_prompts_gateway_id", "prompts", "gateways", ["gateway_id"], ["id"]) -# op.create_foreign_key("fk_server_tool_association_servers", "server_tool_association", "servers", ["server_id"], ["id"]) -# op.create_foreign_key("fk_server_tool_association_tools", "server_tool_association", "tools", ["tool_id"], ["id"]) -# op.create_foreign_key("fk_tool_metrics_tool_id", "tool_metrics", "tools", ["tool_id"], ["id"]) -# op.create_foreign_key("fk_server_metrics_server_id", "server_metrics", "servers", ["server_id"], ["id"]) -# op.create_foreign_key("fk_server_resource_association_server_id", "server_resource_association", "servers", ["server_id"], ["id"]) -# op.create_foreign_key("fk_server_prompt_association_server_id", "server_prompt_association", "servers", ["server_id"], ["id"]) - - -def downgrade() -> None: - """Revert database schema from UUID primary keys back to integers. - - This downgrade reverses the UUID migration but with significant limitations: - - Schema structure is restored but data is NOT preserved - - All UUID values and slug fields are lost - - Foreign key relationships are broken (columns will be NULL) - - Original integer IDs cannot be recovered - - The downgrade operates in reverse order of the upgrade: - - Stage 1 - Revert schema changes: - - Drops UUID-based constraints and keys - - Renames UUID columns back to temporary names - - Re-adds integer columns (empty/NULL) - - Stage 2 - Data migration (skipped): - - Original integer IDs cannot be restored from UUIDs - - Relationships cannot be reconstructed - - Stage 3 - Remove temporary columns: - - Drops all UUID and slug columns - - Leaves database with original schema but no data - - Warning: - This downgrade is destructive and should only be used if you need - to revert the schema structure. All data in affected tables will - need to be manually restored from backups. - - Examples: - >>> # Running the downgrade - >>> downgrade() # doctest: +SKIP - # Schema reverted but data is lost - """ - # ── STAGE 1 (REVERSE): Revert Schema to original state ───────────────── - # This reverses the operations from STAGE 3 of the upgrade. - # Data from the new columns will be lost, which is expected. - - with op.batch_alter_table("server_tool_association") as batch_op: - # Drop new constraints - batch_op.drop_constraint("fk_server_tool_association_tools", type_="foreignkey") - batch_op.drop_constraint("fk_server_tool_association_servers", type_="foreignkey") - batch_op.drop_constraint("pk_server_tool_association", type_="primarykey") - # Rename final columns back to temporary names - batch_op.alter_column("server_id", new_column_name="server_id_new") - batch_op.alter_column("tool_id", new_column_name="tool_id_new") - # Add back old integer columns (data is not restored) - batch_op.add_column(sa.Column("server_id", sa.Integer(), nullable=True)) - batch_op.add_column(sa.Column("tool_id", sa.Integer(), nullable=True)) - - with op.batch_alter_table("tools") as batch_op: - # Drop new constraints - batch_op.drop_constraint("fk_tools_gateway_id", type_="foreignkey") - batch_op.drop_constraint("uq_gateway_id__original_name", type_="unique") - batch_op.drop_constraint("uq_tools_name", type_="unique") - batch_op.drop_constraint("pk_tools", type_="primarykey") - # Rename final columns back to temporary names - batch_op.alter_column("id", new_column_name="id_new") - batch_op.alter_column("gateway_id", new_column_name="gateway_id_new") - batch_op.alter_column("name", new_column_name="name_new") - # Add back old columns - batch_op.add_column(sa.Column("id", sa.Integer(), nullable=True)) - batch_op.add_column(sa.Column("gateway_id", sa.Integer(), nullable=True)) - batch_op.add_column(sa.Column("name", sa.String(), nullable=True)) - - with op.batch_alter_table("servers") as batch_op: - batch_op.drop_constraint("pk_servers", type_="primarykey") - batch_op.alter_column("id", new_column_name="id_new") - batch_op.add_column(sa.Column("id", sa.Integer(), nullable=True)) - - with op.batch_alter_table("gateways") as batch_op: - batch_op.drop_constraint("uq_gateways_url", type_="unique") - batch_op.drop_constraint("uq_gateways_slug", type_="unique") - batch_op.drop_constraint("pk_gateways", type_="primarykey") - batch_op.alter_column("id", new_column_name="id_new") - batch_op.add_column(sa.Column("id", sa.Integer(), nullable=True)) - - # ── STAGE 2 (REVERSE): Reverse Data Migration (No-Op for Schema) ────── - # Reversing the data population (e.g., creating integer PKs from UUIDs) - # is a complex, stateful operation and is omitted here. At this point, - # the original columns exist but are empty (NULL). - - # ── STAGE 3 (REVERSE): Drop the temporary/new columns ──────────────── - # This reverses the operations from STAGE 1 of the upgrade. - op.drop_column("server_tool_association", "tool_id_new") - op.drop_column("server_tool_association", "server_id_new") - op.drop_column("servers", "id_new") - op.drop_column("tools", "gateway_id_new") - op.drop_column("tools", "name_new") - op.drop_column("tools", "original_name_slug") - op.drop_column("tools", "original_name") - op.drop_column("tools", "id_new") - op.drop_column("gateways", "id_new") - op.drop_column("gateways", "slug") diff --git a/mcpgateway/alembic/versions/e4fc04d1a442_add_annotations_to_tables.py b/mcpgateway/alembic/versions/e4fc04d1a442_add_annotations_to_tables.py deleted file mode 100644 index 8876f3b4..00000000 --- a/mcpgateway/alembic/versions/e4fc04d1a442_add_annotations_to_tables.py +++ /dev/null @@ -1,56 +0,0 @@ -# -*- coding: utf-8 -*- -"""Add annotations to tables - -Revision ID: e4fc04d1a442 -Revises: b77ca9d2de7e -Create Date: 2025-06-27 21:45:35.099713 - -""" - -# Standard -from typing import Sequence, Union - -# Third-Party -from alembic import op -import sqlalchemy as sa - -# revision identifiers, used by Alembic. -revision: str = "e4fc04d1a442" -down_revision: Union[str, Sequence[str], None] = "b77ca9d2de7e" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - """ - Applies the migration to add the 'annotations' column. - - This function adds a new column named 'annotations' of type JSON to the 'tool' - table. It includes a server-side default of an empty JSON object ('{}') to ensure - that existing rows get a non-null default value. - """ - bind = op.get_bind() - inspector = sa.inspect(bind) - - if not inspector.has_table("gateways"): - print("Fresh database detected. Skipping migration.") - return - - op.add_column("tools", sa.Column("annotations", sa.JSON(), server_default=sa.text("'{}'"), nullable=False)) - - -def downgrade() -> None: - """ - Reverts the migration by removing the 'annotations' column. - - This function provides a way to undo the migration, safely removing the - 'annotations' column from the 'tool' table. - """ - bind = op.get_bind() - inspector = sa.inspect(bind) - - if not inspector.has_table("gateways"): - print("Fresh database detected. Skipping migration.") - return - - op.drop_column("tools", "annotations") diff --git a/mcpgateway/alembic/versions/e75490e949b1_add_improved_status_to_tables.py b/mcpgateway/alembic/versions/e75490e949b1_add_improved_status_to_tables.py deleted file mode 100644 index 097535b4..00000000 --- a/mcpgateway/alembic/versions/e75490e949b1_add_improved_status_to_tables.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- -"""Add enabled and reachable columns in tools and gateways tables and migrate data (is_active ➜ enabled,reachable). - -Revision ID: e75490e949b1 -Revises: e4fc04d1a442 -Create Date: 2025-07-02 17:12:40.678256 -""" - -# Standard -from typing import Sequence, Union - -# Third-Party -from alembic import op -import sqlalchemy as sa - -# Revision identifiers. -revision: str = "e75490e949b1" -down_revision: Union[str, Sequence[str], None] = "e4fc04d1a442" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade(): - """ - Renames 'is_active' to 'enabled' and adds a new 'reachable' column (default True) - in both 'tools' and 'gateways' tables. - """ - op.alter_column("tools", "is_active", new_column_name="enabled") - op.add_column("tools", sa.Column("reachable", sa.Boolean(), nullable=False, server_default=sa.true())) - - op.alter_column("gateways", "is_active", new_column_name="enabled") - op.add_column("gateways", sa.Column("reachable", sa.Boolean(), nullable=False, server_default=sa.true())) - - -def downgrade(): - """ - Reverts the changes by renaming 'enabled' back to 'is_active' - and dropping the 'reachable' column in both 'tools' and 'gateways' tables. - """ - op.alter_column("tools", "enabled", new_column_name="is_active") - op.drop_column("tools", "reachable") - - op.alter_column("gateways", "enabled", new_column_name="is_active") - op.drop_column("gateways", "reachable") diff --git a/mcpgateway/bootstrap_db.py b/mcpgateway/bootstrap_db.py index c88d666a..af05d08a 100644 --- a/mcpgateway/bootstrap_db.py +++ b/mcpgateway/bootstrap_db.py @@ -57,12 +57,14 @@ async def main() -> None: insp = inspect(conn) - if "gateways" not in insp.get_table_names(): - logger.info("Empty DB detected - creating baseline schema") - command.stamp(cfg, "head") - Base.metadata.create_all(bind=conn) - else: - command.upgrade(cfg, "head") + command.upgrade(cfg, "head") + + # if "gateways" not in insp.get_table_names(): + # logger.info("Empty DB detected - creating baseline schema") + # command.stamp(cfg, "head") + # Base.metadata.create_all(bind=conn) + # else: + # command.upgrade(cfg, "head") logger.info("Database ready") diff --git a/tests/e2e/test_admin_apis.py b/tests/e2e/test_admin_apis.py index 7430089a..37e83779 100644 --- a/tests/e2e/test_admin_apis.py +++ b/tests/e2e/test_admin_apis.py @@ -72,7 +72,7 @@ # Fixtures # ------------------------- @pytest_asyncio.fixture -async def temp_db(): +async def temp_db(monkeypatch): """ Create a temporary SQLite database for testing. @@ -84,9 +84,9 @@ async def temp_db(): db_fd, db_path = tempfile.mkstemp(suffix=".db") sqlite_url = f"sqlite:///{db_path}" - # monkeypatch.setattr(settings, "database_url", sqlite_url) + monkeypatch.setattr(settings, "database_url", sqlite_url) - # await bootstrap_db() # Ensure the database is bootstrapped + await bootstrap_db() # Ensure the database is bootstrapped # Create engine with SQLite engine = create_engine( @@ -95,7 +95,8 @@ async def temp_db(): poolclass=StaticPool, ) - Base.metadata.create_all(bind=engine) # Create tables + + # Base.metadata.create_all(bind=engine) # Create tables # Create session factory TestSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) From 1a47eb5db7054f4dc24a5887f7a57e81eb4aa236 Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Mon, 21 Jul 2025 17:28:04 +0000 Subject: [PATCH 07/21] remove bootstrap_db.py Signed-off-by: Madhav Kandukuri --- gunicorn.config.py | 8 +++++ mcpgateway/bootstrap_db.py | 73 -------------------------------------- mcpgateway/main.py | 2 -- 3 files changed, 8 insertions(+), 75 deletions(-) delete mode 100644 mcpgateway/bootstrap_db.py diff --git a/gunicorn.config.py b/gunicorn.config.py index 3d56821e..e9d0dd22 100644 --- a/gunicorn.config.py +++ b/gunicorn.config.py @@ -14,6 +14,10 @@ Reference: https://stackoverflow.com/questions/10855197/frequent-worker-timeout """ +from importlib.resources import files +from alembic.config import Config +from alembic import command + # First-Party # Import Pydantic Settings singleton from mcpgateway.config import settings @@ -52,6 +56,10 @@ # server hooks +def on_starting(server): + ini_path = files("mcpgateway").joinpath("alembic.ini") + cfg = Config(str(ini_path)) + command.upgrade(cfg, "head") def when_ready(server): server.log.info("Server is ready. Spawning workers") diff --git a/mcpgateway/bootstrap_db.py b/mcpgateway/bootstrap_db.py deleted file mode 100644 index af05d08a..00000000 --- a/mcpgateway/bootstrap_db.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8 -*- -"""Database bootstrap/upgrade entry-point for MCP Gateway. - -Copyright 2025 -SPDX-License-Identifier: Apache-2.0 -Authors: Madhav Kandukuri - -The script: - -1. Creates a synchronous SQLAlchemy ``Engine`` from ``settings.database_url``. -2. Looks for an *alembic.ini* two levels up from this file to drive migrations. -3. If the database is still empty (no ``gateways`` table), it: - - builds the base schema with ``Base.metadata.create_all()`` - - stamps the migration head so Alembic knows it is up-to-date -4. Otherwise, it applies any outstanding Alembic revisions. -5. Logs a **"Database ready"** message on success. - -It is intended to be invoked via ``python3 -m mcpgateway.bootstrap_db`` or -directly with ``python3 mcpgateway/bootstrap_db.py``. -""" - -# Standard -import asyncio -from importlib.resources import files -import logging - -# Third-Party -from alembic import command -from alembic.config import Config -from sqlalchemy import create_engine, inspect - -# First-Party -from mcpgateway.config import settings -from mcpgateway.db import Base - -logger = logging.getLogger(__name__) - - -async def main() -> None: - """ - Bootstrap or upgrade the database schema, then log readiness. - - Runs `create_all()` + `alembic stamp head` on an empty DB, otherwise just - executes `alembic upgrade head`, leaving application data intact. - - Args: - None - """ - engine = create_engine(settings.database_url) - ini_path = files("mcpgateway").joinpath("alembic.ini") - cfg = Config(str(ini_path)) # path in container - cfg.attributes["configure_logger"] = True - - with engine.begin() as conn: - cfg.attributes["connection"] = conn - cfg.set_main_option("sqlalchemy.url", settings.database_url) - - insp = inspect(conn) - - command.upgrade(cfg, "head") - - # if "gateways" not in insp.get_table_names(): - # logger.info("Empty DB detected - creating baseline schema") - # command.stamp(cfg, "head") - # Base.metadata.create_all(bind=conn) - # else: - # command.upgrade(cfg, "head") - - logger.info("Database ready") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/mcpgateway/main.py b/mcpgateway/main.py index dc745a7e..da5360b6 100644 --- a/mcpgateway/main.py +++ b/mcpgateway/main.py @@ -58,7 +58,6 @@ # First-Party from mcpgateway import __version__ from mcpgateway.admin import admin_router -from mcpgateway.bootstrap_db import main as bootstrap_db from mcpgateway.cache import ResourceCache, SessionRegistry from mcpgateway.config import jsonpath_modifier, settings from mcpgateway.db import refresh_slugs_on_startup, SessionLocal @@ -200,7 +199,6 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]: """ logger.info("Starting MCP Gateway services") try: - await bootstrap_db() await tool_service.initialize() await resource_service.initialize() await prompt_service.initialize() From 9c51df1e7630b18c688a1e3b078e27db5adb7b2c Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Mon, 21 Jul 2025 17:31:17 +0000 Subject: [PATCH 08/21] remove bootstrap_db Signed-off-by: Madhav Kandukuri --- tests/e2e/test_admin_apis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e/test_admin_apis.py b/tests/e2e/test_admin_apis.py index 37e83779..13cbc09e 100644 --- a/tests/e2e/test_admin_apis.py +++ b/tests/e2e/test_admin_apis.py @@ -51,7 +51,6 @@ from sqlalchemy.pool import StaticPool # First-Party -from mcpgateway.bootstrap_db import main as bootstrap_db from mcpgateway.config import settings from mcpgateway.db import Base from mcpgateway.main import app, get_db From a035023a5cbcbc710e1bd8a8af6fa28948a6f669 Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Mon, 21 Jul 2025 17:32:12 +0000 Subject: [PATCH 09/21] remove bootstrap_db in tests Signed-off-by: Madhav Kandukuri --- tests/e2e/test_admin_apis.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/e2e/test_admin_apis.py b/tests/e2e/test_admin_apis.py index 13cbc09e..aeb1deca 100644 --- a/tests/e2e/test_admin_apis.py +++ b/tests/e2e/test_admin_apis.py @@ -85,8 +85,6 @@ async def temp_db(monkeypatch): monkeypatch.setattr(settings, "database_url", sqlite_url) - await bootstrap_db() # Ensure the database is bootstrapped - # Create engine with SQLite engine = create_engine( sqlite_url, From 8edb61d080871d81b0c51e700c314e50401ed76b Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Mon, 21 Jul 2025 17:33:12 +0000 Subject: [PATCH 10/21] don't change test_admin_apis --- tests/e2e/test_admin_apis.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/tests/e2e/test_admin_apis.py b/tests/e2e/test_admin_apis.py index aeb1deca..ed60e8e9 100644 --- a/tests/e2e/test_admin_apis.py +++ b/tests/e2e/test_admin_apis.py @@ -35,7 +35,6 @@ os.environ["MCPGATEWAY_UI_ENABLED"] = "true" # Standard -import logging import tempfile from typing import AsyncGenerator from unittest.mock import patch @@ -46,17 +45,14 @@ from httpx import AsyncClient import pytest import pytest_asyncio -from sqlalchemy import create_engine, inspect +from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import StaticPool # First-Party -from mcpgateway.config import settings from mcpgateway.db import Base from mcpgateway.main import app, get_db -logger = logging.getLogger(__name__) - # pytest.skip("Temporarily disabling this suite", allow_module_level=True) # ------------------------- @@ -71,7 +67,7 @@ # Fixtures # ------------------------- @pytest_asyncio.fixture -async def temp_db(monkeypatch): +async def temp_db(): """ Create a temporary SQLite database for testing. @@ -81,19 +77,16 @@ async def temp_db(monkeypatch): """ # Create temporary file for SQLite database db_fd, db_path = tempfile.mkstemp(suffix=".db") - sqlite_url = f"sqlite:///{db_path}" - - monkeypatch.setattr(settings, "database_url", sqlite_url) # Create engine with SQLite engine = create_engine( - sqlite_url, + f"sqlite:///{db_path}", connect_args={"check_same_thread": False}, poolclass=StaticPool, ) - - # Base.metadata.create_all(bind=engine) # Create tables + # Create all tables + Base.metadata.create_all(bind=engine) # Create session factory TestSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) From ec5e85e7515157628d4afa5cee4c5e1e664c7a54 Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Mon, 21 Jul 2025 18:00:29 +0000 Subject: [PATCH 11/21] testing tests Signed-off-by: Madhav Kandukuri --- tests/e2e/test_admin_apis.py | 72 ++++++++++++++---------------------- 1 file changed, 28 insertions(+), 44 deletions(-) diff --git a/tests/e2e/test_admin_apis.py b/tests/e2e/test_admin_apis.py index ed60e8e9..ad98f59f 100644 --- a/tests/e2e/test_admin_apis.py +++ b/tests/e2e/test_admin_apis.py @@ -35,6 +35,7 @@ os.environ["MCPGATEWAY_UI_ENABLED"] = "true" # Standard +import importlib, sys, os, tempfile, pytest_asyncio import tempfile from typing import AsyncGenerator from unittest.mock import patch @@ -50,8 +51,8 @@ from sqlalchemy.pool import StaticPool # First-Party -from mcpgateway.db import Base -from mcpgateway.main import app, get_db +# from mcpgateway.db import Base +# from mcpgateway.main import app, get_db # pytest.skip("Temporarily disabling this suite", allow_module_level=True) @@ -67,65 +68,48 @@ # Fixtures # ------------------------- @pytest_asyncio.fixture -async def temp_db(): - """ - Create a temporary SQLite database for testing. - - This fixture creates a fresh database for each test, ensuring complete - isolation between tests. The database is automatically cleaned up after - the test completes. - """ - # Create temporary file for SQLite database +async def app_with_temp_db(monkeypatch): + # 1. Spin up a temp SQLite file db_fd, db_path = tempfile.mkstemp(suffix=".db") + sqlite_url = f"sqlite:///{db_path}" - # Create engine with SQLite + # 2. Point the settings **before** importing main + from mcpgateway.config import settings + monkeypatch.setattr(settings, "database_url", sqlite_url, raising=False) + + # 3. (Re)import main so it builds a brand‑new engine/SessionLocal + if "mcpgateway.main" in sys.modules: + importlib.reload(sys.modules["mcpgateway.main"]) + else: + import mcpgateway.main # noqa: F401 + + from mcpgateway.main import app + from mcpgateway.db import SessionLocal, Base # SessionLocal is the new one + + # 4. Create tables (or run Alembic migrations here if you prefer) engine = create_engine( - f"sqlite:///{db_path}", + sqlite_url, connect_args={"check_same_thread": False}, poolclass=StaticPool, ) - - # Create all tables Base.metadata.create_all(bind=engine) - # Create session factory - TestSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) - - # Override the get_db dependency - def override_get_db(): - db = TestSessionLocal() - try: - yield db - finally: - db.close() - - app.dependency_overrides[get_db] = override_get_db - - # Override authentication for all tests - # First-Party + # 5. Override auth in one place from mcpgateway.utils.verify_credentials import require_auth, require_basic_auth + app.dependency_overrides[require_auth] = lambda: TEST_USER + app.dependency_overrides[require_basic_auth] = lambda: TEST_USER - def override_auth(): - return TEST_USER - - app.dependency_overrides[require_auth] = override_auth - app.dependency_overrides[require_basic_auth] = override_auth + yield app # << the FastAPI instance with the right DB behind it - yield engine - - # Cleanup + # 6. Cleanup app.dependency_overrides.clear() os.close(db_fd) os.unlink(db_path) - @pytest_asyncio.fixture -async def client(temp_db) -> AsyncGenerator[AsyncClient, None]: - """Create an async test client with the test database.""" - # Third-Party +async def client(app_with_temp_db): from httpx import ASGITransport, AsyncClient - - transport = ASGITransport(app=app) + transport = ASGITransport(app=app_with_temp_db) async with AsyncClient(transport=transport, base_url="http://test") as ac: yield ac From 2d2710cef7d4f4fef983abaa5350a28f2e319da8 Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Mon, 21 Jul 2025 18:13:25 +0000 Subject: [PATCH 12/21] test changes to tests Signed-off-by: Madhav Kandukuri --- tests/e2e/test_admin_apis.py | 51 +++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/tests/e2e/test_admin_apis.py b/tests/e2e/test_admin_apis.py index ad98f59f..e2289bd4 100644 --- a/tests/e2e/test_admin_apis.py +++ b/tests/e2e/test_admin_apis.py @@ -69,42 +69,51 @@ # ------------------------- @pytest_asyncio.fixture async def app_with_temp_db(monkeypatch): - # 1. Spin up a temp SQLite file - db_fd, db_path = tempfile.mkstemp(suffix=".db") - sqlite_url = f"sqlite:///{db_path}" + # 1. fresh sqlite file + fd, path = tempfile.mkstemp(suffix=".db") + url = f"sqlite:///{path}" - # 2. Point the settings **before** importing main + # 2. point settings at the temp DB *before* anything imports the app from mcpgateway.config import settings - monkeypatch.setattr(settings, "database_url", sqlite_url, raising=False) + monkeypatch.setattr(settings, "database_url", url, raising=False) - # 3. (Re)import main so it builds a brand‑new engine/SessionLocal + # 3. Build a new engine + SessionLocal, patch them into mcpgateway.db + import mcpgateway.db as db_mod + engine = create_engine( + url, + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + TestSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + + monkeypatch.setattr(db_mod, "engine", engine, raising=False) + monkeypatch.setattr(db_mod, "SessionLocal", TestSessionLocal, raising=False) + + # 4. create schema (or run Alembic migrations) + db_mod.Base.metadata.create_all(bind=engine) + + # 5. Now import / reload the FastAPI app **after** the patch if "mcpgateway.main" in sys.modules: importlib.reload(sys.modules["mcpgateway.main"]) else: - import mcpgateway.main # noqa: F401 + import mcpgateway.main # noqa: F401 from mcpgateway.main import app - from mcpgateway.db import SessionLocal, Base # SessionLocal is the new one - - # 4. Create tables (or run Alembic migrations here if you prefer) - engine = create_engine( - sqlite_url, - connect_args={"check_same_thread": False}, - poolclass=StaticPool, + from mcpgateway.utils.verify_credentials import ( + require_auth, + require_basic_auth, ) - Base.metadata.create_all(bind=engine) - # 5. Override auth in one place - from mcpgateway.utils.verify_credentials import require_auth, require_basic_auth + # 6. bypass auth for tests app.dependency_overrides[require_auth] = lambda: TEST_USER app.dependency_overrides[require_basic_auth] = lambda: TEST_USER - yield app # << the FastAPI instance with the right DB behind it + yield app - # 6. Cleanup + # 7. teardown app.dependency_overrides.clear() - os.close(db_fd) - os.unlink(db_path) + os.close(fd) + os.unlink(path) @pytest_asyncio.fixture async def client(app_with_temp_db): From 0e3758e37d324e2d7b91b2eb3880fbcfe0440949 Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Mon, 21 Jul 2025 19:04:21 +0000 Subject: [PATCH 13/21] testing tests Signed-off-by: Madhav Kandukuri --- tests/conftest.py | 51 ++++++++++++ tests/e2e/test_admin_apis.py | 101 ++++++++++++----------- tests/unit/mcpgateway/test_ui_version.py | 73 ++++++---------- 3 files changed, 128 insertions(+), 97 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0eb08fda..e2a503f7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,12 +10,16 @@ # Standard import asyncio import os +import sys +import tempfile from unittest.mock import AsyncMock, patch # Third-Party import pytest +from _pytest.monkeypatch import MonkeyPatch from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool # First-Party from mcpgateway.config import Settings @@ -107,3 +111,50 @@ def mock_websocket(): # import mcpgateway.translate as translate # translate._run_stdio_to_sse = AsyncMock(return_value=None) # translate._run_sse_to_stdio = AsyncMock(return_value=None) + +@pytest.fixture(scope="module") # one DB per test module is usually fine +def app_with_temp_db(): + """Return a FastAPI app wired to a fresh SQLite database.""" + mp = MonkeyPatch() + + # 1) create temp SQLite file + fd, path = tempfile.mkstemp(suffix=".db") + url = f"sqlite:///{path}" + + # 2) patch settings + from mcpgateway.config import settings + mp.setattr(settings, "database_url", url, raising=False) + + import mcpgateway.db as db_mod + engine = create_engine( + url, connect_args={"check_same_thread": False}, poolclass=StaticPool + ) + TestSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + mp.setattr(db_mod, "engine", engine, raising=False) + mp.setattr(db_mod, "SessionLocal", TestSessionLocal, raising=False) + + # 4) patch the already‑imported main module **without reloading** + import mcpgateway.main as main_mod + mp.setattr(main_mod, "SessionLocal", TestSessionLocal, raising=False) + # (patch engine too if your code references it) + mp.setattr(main_mod, "engine", engine, raising=False) + + # 4) create schema + db_mod.Base.metadata.create_all(bind=engine) + + # 5) reload main so routers, deps pick up new SessionLocal + # if "mcpgateway.main" in sys.modules: + # import importlib + + # importlib.reload(sys.modules["mcpgateway.main"]) + # else: + # import mcpgateway.main # noqa: F401 + + from mcpgateway.main import app + yield app + + # 6) teardown + mp.undo() + engine.dispose() + os.close(fd) + os.unlink(path) \ No newline at end of file diff --git a/tests/e2e/test_admin_apis.py b/tests/e2e/test_admin_apis.py index e2289bd4..f9bb02ef 100644 --- a/tests/e2e/test_admin_apis.py +++ b/tests/e2e/test_admin_apis.py @@ -67,61 +67,68 @@ # ------------------------- # Fixtures # ------------------------- -@pytest_asyncio.fixture -async def app_with_temp_db(monkeypatch): - # 1. fresh sqlite file - fd, path = tempfile.mkstemp(suffix=".db") - url = f"sqlite:///{path}" - - # 2. point settings at the temp DB *before* anything imports the app - from mcpgateway.config import settings - monkeypatch.setattr(settings, "database_url", url, raising=False) - - # 3. Build a new engine + SessionLocal, patch them into mcpgateway.db - import mcpgateway.db as db_mod - engine = create_engine( - url, - connect_args={"check_same_thread": False}, - poolclass=StaticPool, - ) - TestSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) - - monkeypatch.setattr(db_mod, "engine", engine, raising=False) - monkeypatch.setattr(db_mod, "SessionLocal", TestSessionLocal, raising=False) - - # 4. create schema (or run Alembic migrations) - db_mod.Base.metadata.create_all(bind=engine) - - # 5. Now import / reload the FastAPI app **after** the patch - if "mcpgateway.main" in sys.modules: - importlib.reload(sys.modules["mcpgateway.main"]) - else: - import mcpgateway.main # noqa: F401 - - from mcpgateway.main import app - from mcpgateway.utils.verify_credentials import ( - require_auth, - require_basic_auth, - ) - - # 6. bypass auth for tests - app.dependency_overrides[require_auth] = lambda: TEST_USER - app.dependency_overrides[require_basic_auth] = lambda: TEST_USER - - yield app - - # 7. teardown - app.dependency_overrides.clear() - os.close(fd) - os.unlink(path) +# @pytest_asyncio.fixture +# async def app_with_temp_db(monkeypatch): +# # 1. fresh sqlite file +# fd, path = tempfile.mkstemp(suffix=".db") +# url = f"sqlite:///{path}" + +# # 2. point settings at the temp DB *before* anything imports the app +# from mcpgateway.config import settings +# monkeypatch.setattr(settings, "database_url", url, raising=False) + +# # 3. Build a new engine + SessionLocal, patch them into mcpgateway.db +# import mcpgateway.db as db_mod +# engine = create_engine( +# url, +# connect_args={"check_same_thread": False}, +# poolclass=StaticPool, +# ) +# TestSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + +# monkeypatch.setattr(db_mod, "engine", engine, raising=False) +# monkeypatch.setattr(db_mod, "SessionLocal", TestSessionLocal, raising=False) + +# # 4. create schema (or run Alembic migrations) +# db_mod.Base.metadata.create_all(bind=engine) + +# # 5. Now import / reload the FastAPI app **after** the patch +# if "mcpgateway.main" in sys.modules: +# importlib.reload(sys.modules["mcpgateway.main"]) +# else: +# import mcpgateway.main # noqa: F401 + +# from mcpgateway.main import app +# from mcpgateway.utils.verify_credentials import ( +# require_auth, +# require_basic_auth, +# ) + +# # 6. bypass auth for tests +# app.dependency_overrides[require_auth] = lambda: TEST_USER +# app.dependency_overrides[require_basic_auth] = lambda: TEST_USER + +# yield app + +# # 7. teardown +# app.dependency_overrides.clear() +# os.close(fd) +# os.unlink(path) @pytest_asyncio.fixture async def client(app_with_temp_db): + from mcpgateway.utils.verify_credentials import require_auth, require_basic_auth + app_with_temp_db.dependency_overrides[require_auth] = lambda: TEST_USER + app_with_temp_db.dependency_overrides[require_basic_auth] = lambda: TEST_USER + from httpx import ASGITransport, AsyncClient transport = ASGITransport(app=app_with_temp_db) async with AsyncClient(transport=transport, base_url="http://test") as ac: yield ac + app_with_temp_db.dependency_overrides.pop(require_auth, None) + app_with_temp_db.dependency_overrides.pop(require_basic_auth, None) + @pytest_asyncio.fixture async def mock_settings(): diff --git a/tests/unit/mcpgateway/test_ui_version.py b/tests/unit/mcpgateway/test_ui_version.py index 5d4b68b6..8e95f41f 100644 --- a/tests/unit/mcpgateway/test_ui_version.py +++ b/tests/unit/mcpgateway/test_ui_version.py @@ -7,82 +7,55 @@ Author: Mihai Criveti """ -# Future from __future__ import annotations # Standard import base64 from typing import Dict -# Third-Party +# Third‑Party import pytest from starlette.testclient import TestClient -# First-Party +# First‑Party from mcpgateway.config import settings -from mcpgateway.main import app -# --------------------------------------------------------------------------- # -# Fixtures -# --------------------------------------------------------------------------- # -@pytest.fixture(scope="session") -def test_client() -> TestClient: - """Spin up the FastAPI test client once for the whole session.""" - return TestClient(app) +# ────────────────────────────────────────────── +# Fixtures (local to this test file) +# ────────────────────────────────────────────── +@pytest.fixture(scope="module") +def test_client(app_with_temp_db) -> TestClient: + """ + Build a TestClient against the FastAPI app that + app_with_temp_db returns (i.e. the one wired to + the temporary SQLite database). + """ + return TestClient(app_with_temp_db) @pytest.fixture() def auth_headers() -> Dict[str, str]: - """ - Build the auth headers expected by the gateway: - - * Authorization: Basic - * X-API-Key: user:pw (plain text) - """ creds = f"{settings.basic_auth_user}:{settings.basic_auth_password}" basic_b64 = base64.b64encode(creds.encode()).decode() - return { "Authorization": f"Basic {basic_b64}", "X-API-Key": creds, } -# --------------------------------------------------------------------------- # +# ────────────────────────────────────────────── # Tests -# --------------------------------------------------------------------------- # -# def test_version_partial_html(test_client: TestClient, auth_headers: Dict[str, str]): -# """ -# /version?partial=true must return an HTML fragment with core meta-info. -# """ -# resp = test_client.get("/version?partial=true", headers=auth_headers) -# assert resp.status_code == 200 -# assert "text/html" in resp.headers["content-type"] - -# html = resp.text -# # Very loose sanity checks - we only care that it is an HTML fragment -# # and that some well-known marker exists. -# assert " Date: Mon, 21 Jul 2025 19:13:14 +0000 Subject: [PATCH 14/21] linting fixes Signed-off-by: Madhav Kandukuri --- gunicorn.config.py | 5 +- .../versions/9c07b4bc5774_auto_migration.py | 414 ++++++++++-------- tests/conftest.py | 20 +- tests/e2e/test_admin_apis.py | 12 +- tests/unit/mcpgateway/test_ui_version.py | 3 +- 5 files changed, 262 insertions(+), 192 deletions(-) diff --git a/gunicorn.config.py b/gunicorn.config.py index e9d0dd22..d3794acc 100644 --- a/gunicorn.config.py +++ b/gunicorn.config.py @@ -14,9 +14,12 @@ Reference: https://stackoverflow.com/questions/10855197/frequent-worker-timeout """ +# Standard from importlib.resources import files -from alembic.config import Config + +# Third-Party from alembic import command +from alembic.config import Config # First-Party # Import Pydantic Settings singleton diff --git a/mcpgateway/alembic/versions/9c07b4bc5774_auto_migration.py b/mcpgateway/alembic/versions/9c07b4bc5774_auto_migration.py index 8cfc5c59..01f7d5d7 100644 --- a/mcpgateway/alembic/versions/9c07b4bc5774_auto_migration.py +++ b/mcpgateway/alembic/versions/9c07b4bc5774_auto_migration.py @@ -1,18 +1,20 @@ """auto migration Revision ID: 9c07b4bc5774 -Revises: +Revises: Create Date: 2025-07-21 17:05:00.624436 """ + +# Standard from typing import Sequence, Union +# Third-Party from alembic import op import sqlalchemy as sa - # revision identifiers, used by Alembic. -revision: str = '9c07b4bc5774' +revision: str = "9c07b4bc5774" down_revision: Union[str, Sequence[str], None] = None branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None @@ -21,178 +23,238 @@ def upgrade() -> None: """Upgrade schema.""" # ### commands auto generated by Alembic - please adjust! ### - op.create_table('gateways', - sa.Column('id', sa.String(length=36), nullable=False), - sa.Column('name', sa.String(), nullable=False), - sa.Column('slug', sa.String(), nullable=False), - sa.Column('url', sa.String(), nullable=False), - sa.Column('description', sa.String(), nullable=True), - sa.Column('transport', sa.String(), nullable=False), - sa.Column('capabilities', sa.JSON(), nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), - sa.Column('enabled', sa.Boolean(), nullable=False), - sa.Column('reachable', sa.Boolean(), nullable=False), - sa.Column('last_seen', sa.DateTime(), nullable=True), - sa.Column('auth_type', sa.String(), nullable=True), - sa.Column('auth_value', sa.JSON(), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('slug'), - sa.UniqueConstraint('url') + op.create_table( + "gateways", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("slug", sa.String(), nullable=False), + sa.Column("url", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column("transport", sa.String(), nullable=False), + sa.Column("capabilities", sa.JSON(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("enabled", sa.Boolean(), nullable=False), + sa.Column("reachable", sa.Boolean(), nullable=False), + sa.Column("last_seen", sa.DateTime(), nullable=True), + sa.Column("auth_type", sa.String(), nullable=True), + sa.Column("auth_value", sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("slug"), + sa.UniqueConstraint("url"), ) - op.create_table('mcp_sessions', - sa.Column('session_id', sa.String(), nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), - sa.Column('last_accessed', sa.DateTime(timezone=True), nullable=False), - sa.Column('data', sa.String(), nullable=True), - sa.PrimaryKeyConstraint('session_id') + op.create_table( + "mcp_sessions", + sa.Column("session_id", sa.String(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("last_accessed", sa.DateTime(timezone=True), nullable=False), + sa.Column("data", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("session_id"), ) - op.create_table('servers', - sa.Column('id', sa.String(length=36), nullable=False), - sa.Column('name', sa.String(), nullable=False), - sa.Column('description', sa.String(), nullable=True), - sa.Column('icon', sa.String(), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), - sa.Column('is_active', sa.Boolean(), nullable=False), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('name') + op.create_table( + "servers", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column("icon", sa.String(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("is_active", sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("name"), ) - op.create_table('mcp_messages', - sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), - sa.Column('session_id', sa.String(), nullable=False), - sa.Column('message', sa.String(), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), - sa.Column('last_accessed', sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(['session_id'], ['mcp_sessions.session_id'], ), - sa.PrimaryKeyConstraint('id') + op.create_table( + "mcp_messages", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("session_id", sa.String(), nullable=False), + sa.Column("message", sa.String(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("last_accessed", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint( + ["session_id"], + ["mcp_sessions.session_id"], + ), + sa.PrimaryKeyConstraint("id"), ) - op.create_table('prompts', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('name', sa.String(), nullable=False), - sa.Column('description', sa.String(), nullable=True), - sa.Column('template', sa.Text(), nullable=False), - sa.Column('argument_schema', sa.JSON(), nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), - sa.Column('is_active', sa.Boolean(), nullable=False), - sa.Column('gateway_id', sa.String(length=36), nullable=True), - sa.ForeignKeyConstraint(['gateway_id'], ['gateways.id'], ), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('name') + op.create_table( + "prompts", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column("template", sa.Text(), nullable=False), + sa.Column("argument_schema", sa.JSON(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("is_active", sa.Boolean(), nullable=False), + sa.Column("gateway_id", sa.String(length=36), nullable=True), + sa.ForeignKeyConstraint( + ["gateway_id"], + ["gateways.id"], + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("name"), ) - op.create_table('resources', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('uri', sa.String(), nullable=False), - sa.Column('name', sa.String(), nullable=False), - sa.Column('description', sa.String(), nullable=True), - sa.Column('mime_type', sa.String(), nullable=True), - sa.Column('size', sa.Integer(), nullable=True), - sa.Column('template', sa.String(), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), - sa.Column('is_active', sa.Boolean(), nullable=False), - sa.Column('text_content', sa.Text(), nullable=True), - sa.Column('binary_content', sa.LargeBinary(), nullable=True), - sa.Column('gateway_id', sa.String(length=36), nullable=True), - sa.ForeignKeyConstraint(['gateway_id'], ['gateways.id'], ), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('uri') + op.create_table( + "resources", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("uri", sa.String(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column("mime_type", sa.String(), nullable=True), + sa.Column("size", sa.Integer(), nullable=True), + sa.Column("template", sa.String(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("is_active", sa.Boolean(), nullable=False), + sa.Column("text_content", sa.Text(), nullable=True), + sa.Column("binary_content", sa.LargeBinary(), nullable=True), + sa.Column("gateway_id", sa.String(length=36), nullable=True), + sa.ForeignKeyConstraint( + ["gateway_id"], + ["gateways.id"], + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("uri"), ) - op.create_table('server_metrics', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('server_id', sa.String(), nullable=False), - sa.Column('timestamp', sa.DateTime(timezone=True), nullable=False), - sa.Column('response_time', sa.Float(), nullable=False), - sa.Column('is_success', sa.Boolean(), nullable=False), - sa.Column('error_message', sa.Text(), nullable=True), - sa.ForeignKeyConstraint(['server_id'], ['servers.id'], ), - sa.PrimaryKeyConstraint('id') + op.create_table( + "server_metrics", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("server_id", sa.String(), nullable=False), + sa.Column("timestamp", sa.DateTime(timezone=True), nullable=False), + sa.Column("response_time", sa.Float(), nullable=False), + sa.Column("is_success", sa.Boolean(), nullable=False), + sa.Column("error_message", sa.Text(), nullable=True), + sa.ForeignKeyConstraint( + ["server_id"], + ["servers.id"], + ), + sa.PrimaryKeyConstraint("id"), ) - op.create_table('tools', - sa.Column('id', sa.String(length=36), nullable=False), - sa.Column('original_name', sa.String(), nullable=False), - sa.Column('original_name_slug', sa.String(), nullable=False), - sa.Column('url', sa.String(), nullable=True), - sa.Column('description', sa.String(), nullable=True), - sa.Column('integration_type', sa.String(), nullable=False), - sa.Column('request_type', sa.String(), nullable=False), - sa.Column('headers', sa.JSON(), nullable=True), - sa.Column('input_schema', sa.JSON(), nullable=False), - sa.Column('annotations', sa.JSON(), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), - sa.Column('enabled', sa.Boolean(), nullable=False), - sa.Column('reachable', sa.Boolean(), nullable=False), - sa.Column('jsonpath_filter', sa.String(), nullable=False), - sa.Column('auth_type', sa.String(), nullable=True), - sa.Column('auth_value', sa.String(), nullable=True), - sa.Column('gateway_id', sa.String(length=36), nullable=True), - sa.Column('name', sa.String(), nullable=True), - sa.ForeignKeyConstraint(['gateway_id'], ['gateways.id'], ), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('gateway_id', 'original_name', name='uq_gateway_id__original_name'), - sa.UniqueConstraint('name') + op.create_table( + "tools", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("original_name", sa.String(), nullable=False), + sa.Column("original_name_slug", sa.String(), nullable=False), + sa.Column("url", sa.String(), nullable=True), + sa.Column("description", sa.String(), nullable=True), + sa.Column("integration_type", sa.String(), nullable=False), + sa.Column("request_type", sa.String(), nullable=False), + sa.Column("headers", sa.JSON(), nullable=True), + sa.Column("input_schema", sa.JSON(), nullable=False), + sa.Column("annotations", sa.JSON(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("enabled", sa.Boolean(), nullable=False), + sa.Column("reachable", sa.Boolean(), nullable=False), + sa.Column("jsonpath_filter", sa.String(), nullable=False), + sa.Column("auth_type", sa.String(), nullable=True), + sa.Column("auth_value", sa.String(), nullable=True), + sa.Column("gateway_id", sa.String(length=36), nullable=True), + sa.Column("name", sa.String(), nullable=True), + sa.ForeignKeyConstraint( + ["gateway_id"], + ["gateways.id"], + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("gateway_id", "original_name", name="uq_gateway_id__original_name"), + sa.UniqueConstraint("name"), ) - op.create_table('prompt_metrics', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('prompt_id', sa.Integer(), nullable=False), - sa.Column('timestamp', sa.DateTime(timezone=True), nullable=False), - sa.Column('response_time', sa.Float(), nullable=False), - sa.Column('is_success', sa.Boolean(), nullable=False), - sa.Column('error_message', sa.Text(), nullable=True), - sa.ForeignKeyConstraint(['prompt_id'], ['prompts.id'], ), - sa.PrimaryKeyConstraint('id') + op.create_table( + "prompt_metrics", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("prompt_id", sa.Integer(), nullable=False), + sa.Column("timestamp", sa.DateTime(timezone=True), nullable=False), + sa.Column("response_time", sa.Float(), nullable=False), + sa.Column("is_success", sa.Boolean(), nullable=False), + sa.Column("error_message", sa.Text(), nullable=True), + sa.ForeignKeyConstraint( + ["prompt_id"], + ["prompts.id"], + ), + sa.PrimaryKeyConstraint("id"), ) - op.create_table('resource_metrics', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('resource_id', sa.Integer(), nullable=False), - sa.Column('timestamp', sa.DateTime(timezone=True), nullable=False), - sa.Column('response_time', sa.Float(), nullable=False), - sa.Column('is_success', sa.Boolean(), nullable=False), - sa.Column('error_message', sa.Text(), nullable=True), - sa.ForeignKeyConstraint(['resource_id'], ['resources.id'], ), - sa.PrimaryKeyConstraint('id') + op.create_table( + "resource_metrics", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("resource_id", sa.Integer(), nullable=False), + sa.Column("timestamp", sa.DateTime(timezone=True), nullable=False), + sa.Column("response_time", sa.Float(), nullable=False), + sa.Column("is_success", sa.Boolean(), nullable=False), + sa.Column("error_message", sa.Text(), nullable=True), + sa.ForeignKeyConstraint( + ["resource_id"], + ["resources.id"], + ), + sa.PrimaryKeyConstraint("id"), ) - op.create_table('resource_subscriptions', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('resource_id', sa.Integer(), nullable=False), - sa.Column('subscriber_id', sa.String(), nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), - sa.Column('last_notification', sa.DateTime(), nullable=True), - sa.ForeignKeyConstraint(['resource_id'], ['resources.id'], ), - sa.PrimaryKeyConstraint('id') + op.create_table( + "resource_subscriptions", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("resource_id", sa.Integer(), nullable=False), + sa.Column("subscriber_id", sa.String(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("last_notification", sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint( + ["resource_id"], + ["resources.id"], + ), + sa.PrimaryKeyConstraint("id"), ) - op.create_table('server_prompt_association', - sa.Column('server_id', sa.String(), nullable=False), - sa.Column('prompt_id', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['prompt_id'], ['prompts.id'], ), - sa.ForeignKeyConstraint(['server_id'], ['servers.id'], ), - sa.PrimaryKeyConstraint('server_id', 'prompt_id') + op.create_table( + "server_prompt_association", + sa.Column("server_id", sa.String(), nullable=False), + sa.Column("prompt_id", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ["prompt_id"], + ["prompts.id"], + ), + sa.ForeignKeyConstraint( + ["server_id"], + ["servers.id"], + ), + sa.PrimaryKeyConstraint("server_id", "prompt_id"), ) - op.create_table('server_resource_association', - sa.Column('server_id', sa.String(), nullable=False), - sa.Column('resource_id', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['resource_id'], ['resources.id'], ), - sa.ForeignKeyConstraint(['server_id'], ['servers.id'], ), - sa.PrimaryKeyConstraint('server_id', 'resource_id') + op.create_table( + "server_resource_association", + sa.Column("server_id", sa.String(), nullable=False), + sa.Column("resource_id", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ["resource_id"], + ["resources.id"], + ), + sa.ForeignKeyConstraint( + ["server_id"], + ["servers.id"], + ), + sa.PrimaryKeyConstraint("server_id", "resource_id"), ) - op.create_table('server_tool_association', - sa.Column('server_id', sa.String(), nullable=False), - sa.Column('tool_id', sa.String(), nullable=False), - sa.ForeignKeyConstraint(['server_id'], ['servers.id'], ), - sa.ForeignKeyConstraint(['tool_id'], ['tools.id'], ), - sa.PrimaryKeyConstraint('server_id', 'tool_id') + op.create_table( + "server_tool_association", + sa.Column("server_id", sa.String(), nullable=False), + sa.Column("tool_id", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["server_id"], + ["servers.id"], + ), + sa.ForeignKeyConstraint( + ["tool_id"], + ["tools.id"], + ), + sa.PrimaryKeyConstraint("server_id", "tool_id"), ) - op.create_table('tool_metrics', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('tool_id', sa.String(), nullable=False), - sa.Column('timestamp', sa.DateTime(timezone=True), nullable=False), - sa.Column('response_time', sa.Float(), nullable=False), - sa.Column('is_success', sa.Boolean(), nullable=False), - sa.Column('error_message', sa.Text(), nullable=True), - sa.ForeignKeyConstraint(['tool_id'], ['tools.id'], ), - sa.PrimaryKeyConstraint('id') + op.create_table( + "tool_metrics", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("tool_id", sa.String(), nullable=False), + sa.Column("timestamp", sa.DateTime(timezone=True), nullable=False), + sa.Column("response_time", sa.Float(), nullable=False), + sa.Column("is_success", sa.Boolean(), nullable=False), + sa.Column("error_message", sa.Text(), nullable=True), + sa.ForeignKeyConstraint( + ["tool_id"], + ["tools.id"], + ), + sa.PrimaryKeyConstraint("id"), ) # ### end Alembic commands ### @@ -200,19 +262,19 @@ def upgrade() -> None: def downgrade() -> None: """Downgrade schema.""" # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('tool_metrics') - op.drop_table('server_tool_association') - op.drop_table('server_resource_association') - op.drop_table('server_prompt_association') - op.drop_table('resource_subscriptions') - op.drop_table('resource_metrics') - op.drop_table('prompt_metrics') - op.drop_table('tools') - op.drop_table('server_metrics') - op.drop_table('resources') - op.drop_table('prompts') - op.drop_table('mcp_messages') - op.drop_table('servers') - op.drop_table('mcp_sessions') - op.drop_table('gateways') + op.drop_table("tool_metrics") + op.drop_table("server_tool_association") + op.drop_table("server_resource_association") + op.drop_table("server_prompt_association") + op.drop_table("resource_subscriptions") + op.drop_table("resource_metrics") + op.drop_table("prompt_metrics") + op.drop_table("tools") + op.drop_table("server_metrics") + op.drop_table("resources") + op.drop_table("prompts") + op.drop_table("mcp_messages") + op.drop_table("servers") + op.drop_table("mcp_sessions") + op.drop_table("gateways") # ### end Alembic commands ### diff --git a/tests/conftest.py b/tests/conftest.py index e2a503f7..73d4cc5b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,13 +10,12 @@ # Standard import asyncio import os -import sys import tempfile from unittest.mock import AsyncMock, patch # Third-Party -import pytest from _pytest.monkeypatch import MonkeyPatch +import pytest from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import StaticPool @@ -112,7 +111,8 @@ def mock_websocket(): # translate._run_stdio_to_sse = AsyncMock(return_value=None) # translate._run_sse_to_stdio = AsyncMock(return_value=None) -@pytest.fixture(scope="module") # one DB per test module is usually fine + +@pytest.fixture(scope="module") # one DB per test module is usually fine def app_with_temp_db(): """Return a FastAPI app wired to a fresh SQLite database.""" mp = MonkeyPatch() @@ -122,19 +122,23 @@ def app_with_temp_db(): url = f"sqlite:///{path}" # 2) patch settings + # First-Party from mcpgateway.config import settings + mp.setattr(settings, "database_url", url, raising=False) + # First-Party import mcpgateway.db as db_mod - engine = create_engine( - url, connect_args={"check_same_thread": False}, poolclass=StaticPool - ) + + engine = create_engine(url, connect_args={"check_same_thread": False}, poolclass=StaticPool) TestSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) mp.setattr(db_mod, "engine", engine, raising=False) mp.setattr(db_mod, "SessionLocal", TestSessionLocal, raising=False) # 4) patch the already‑imported main module **without reloading** + # First-Party import mcpgateway.main as main_mod + mp.setattr(main_mod, "SessionLocal", TestSessionLocal, raising=False) # (patch engine too if your code references it) mp.setattr(main_mod, "engine", engine, raising=False) @@ -150,11 +154,13 @@ def app_with_temp_db(): # else: # import mcpgateway.main # noqa: F401 + # First-Party from mcpgateway.main import app + yield app # 6) teardown mp.undo() engine.dispose() os.close(fd) - os.unlink(path) \ No newline at end of file + os.unlink(path) diff --git a/tests/e2e/test_admin_apis.py b/tests/e2e/test_admin_apis.py index f9bb02ef..bc1373bb 100644 --- a/tests/e2e/test_admin_apis.py +++ b/tests/e2e/test_admin_apis.py @@ -35,9 +35,6 @@ os.environ["MCPGATEWAY_UI_ENABLED"] = "true" # Standard -import importlib, sys, os, tempfile, pytest_asyncio -import tempfile -from typing import AsyncGenerator from unittest.mock import patch from urllib.parse import quote import uuid @@ -46,11 +43,7 @@ from httpx import AsyncClient import pytest import pytest_asyncio -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from sqlalchemy.pool import StaticPool -# First-Party # from mcpgateway.db import Base # from mcpgateway.main import app, get_db @@ -115,13 +108,18 @@ # os.close(fd) # os.unlink(path) + @pytest_asyncio.fixture async def client(app_with_temp_db): + # First-Party from mcpgateway.utils.verify_credentials import require_auth, require_basic_auth + app_with_temp_db.dependency_overrides[require_auth] = lambda: TEST_USER app_with_temp_db.dependency_overrides[require_basic_auth] = lambda: TEST_USER + # Third-Party from httpx import ASGITransport, AsyncClient + transport = ASGITransport(app=app_with_temp_db) async with AsyncClient(transport=transport, base_url="http://test") as ac: yield ac diff --git a/tests/unit/mcpgateway/test_ui_version.py b/tests/unit/mcpgateway/test_ui_version.py index 8e95f41f..1d88cf29 100644 --- a/tests/unit/mcpgateway/test_ui_version.py +++ b/tests/unit/mcpgateway/test_ui_version.py @@ -7,6 +7,7 @@ Author: Mihai Criveti """ +# Future from __future__ import annotations # Standard @@ -58,4 +59,4 @@ def test_admin_ui_contains_version_tab(test_client: TestClient, auth_headers: Di resp = test_client.get("/admin", headers=auth_headers) assert resp.status_code == 200 assert 'id="tab-version-info"' in resp.text - assert "Version and Environment Info" in resp.text \ No newline at end of file + assert "Version and Environment Info" in resp.text From 1361cc692480344d372e2f0e97c3d23aaf3ca03d Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Tue, 22 Jul 2025 17:03:44 +0000 Subject: [PATCH 15/21] Remove commented code Signed-off-by: Madhav Kandukuri --- tests/e2e/test_admin_apis.py | 49 ------------------------------------ 1 file changed, 49 deletions(-) diff --git a/tests/e2e/test_admin_apis.py b/tests/e2e/test_admin_apis.py index bfd5a67a..30fcdac1 100644 --- a/tests/e2e/test_admin_apis.py +++ b/tests/e2e/test_admin_apis.py @@ -70,55 +70,6 @@ def setup_logging(): # ------------------------- # Fixtures # ------------------------- -# @pytest_asyncio.fixture -# async def app_with_temp_db(monkeypatch): -# # 1. fresh sqlite file -# fd, path = tempfile.mkstemp(suffix=".db") -# url = f"sqlite:///{path}" - -# # 2. point settings at the temp DB *before* anything imports the app -# from mcpgateway.config import settings -# monkeypatch.setattr(settings, "database_url", url, raising=False) - -# # 3. Build a new engine + SessionLocal, patch them into mcpgateway.db -# import mcpgateway.db as db_mod -# engine = create_engine( -# url, -# connect_args={"check_same_thread": False}, -# poolclass=StaticPool, -# ) -# TestSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) - -# monkeypatch.setattr(db_mod, "engine", engine, raising=False) -# monkeypatch.setattr(db_mod, "SessionLocal", TestSessionLocal, raising=False) - -# # 4. create schema (or run Alembic migrations) -# db_mod.Base.metadata.create_all(bind=engine) - -# # 5. Now import / reload the FastAPI app **after** the patch -# if "mcpgateway.main" in sys.modules: -# importlib.reload(sys.modules["mcpgateway.main"]) -# else: -# import mcpgateway.main # noqa: F401 - -# from mcpgateway.main import app -# from mcpgateway.utils.verify_credentials import ( -# require_auth, -# require_basic_auth, -# ) - -# # 6. bypass auth for tests -# app.dependency_overrides[require_auth] = lambda: TEST_USER -# app.dependency_overrides[require_basic_auth] = lambda: TEST_USER - -# yield app - -# # 7. teardown -# app.dependency_overrides.clear() -# os.close(fd) -# os.unlink(path) - - @pytest_asyncio.fixture async def client(app_with_temp_db): # First-Party From 42a19cb7cd26e09458132e2b565175d30f7954ea Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Thu, 31 Jul 2025 13:56:01 +0530 Subject: [PATCH 16/21] Run migration from docker compose Signed-off-by: Madhav Kandukuri --- docker-compose.yml | 38 +++++++++++++++----------------------- gunicorn.config.py | 6 ------ 2 files changed, 15 insertions(+), 29 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index fd60db07..2cc9041e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,16 +60,8 @@ services: condition: service_healthy # β–Ά wait for DB redis: condition: service_started - # migration: - # condition: service_completed_successfully - - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:4444/health"] - #test: ["CMD", "curl", "-f", "https://localhost:4444/health"] - interval: 30s - timeout: 10s - retries: 5 - start_period: 20s + migration: + condition: service_completed_successfully # volumes: # - ./certs:/app/certs:ro # mount certs folder read-only @@ -123,19 +115,19 @@ services: # volumes: [mongodata:/data/db] # networks: [mcpnet] - # migration: - # #image: ghcr.io/ibm/mcp-context-forge:0.4.0 # Use the release MCP Context Forge image - # image: mcpgateway/mcpgateway:latest # Use the local latest image. Run `make docker-prod` to build it. - # build: - # context: . - # dockerfile: Containerfile - # environment: - # - DATABASE_URL=postgresql://postgres:${POSTGRES_PASSWORD:-mysecretpassword}@postgres:5432/mcp - # command: alembic upgrade head - # depends_on: - # postgres: - # condition: service_healthy - # networks: [mcpnet] + migration: + #image: ghcr.io/ibm/mcp-context-forge:0.4.0 # Use the release MCP Context Forge image + image: mcpgateway/mcpgateway:latest # Use the local latest image. Run `make docker-prod` to build it. + build: + context: . + dockerfile: Containerfile.lite + environment: + - DATABASE_URL=postgresql://postgres:${POSTGRES_PASSWORD:-mysecretpassword}@postgres:5432/mcp + command: alembic -c mcpgateway/alembic.ini upgrade head + depends_on: + postgres: + condition: service_healthy + networks: [mcpnet] ############################################################################### # CACHE diff --git a/gunicorn.config.py b/gunicorn.config.py index d3794acc..ae5716bf 100644 --- a/gunicorn.config.py +++ b/gunicorn.config.py @@ -58,12 +58,6 @@ # ca-certs = '/etc/ca_bundle.crt' # server hooks - -def on_starting(server): - ini_path = files("mcpgateway").joinpath("alembic.ini") - cfg = Config(str(ini_path)) - command.upgrade(cfg, "head") - def when_ready(server): server.log.info("Server is ready. Spawning workers") From 38058a0336f32496547406b8d0a59080d5028563 Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Thu, 31 Jul 2025 13:58:02 +0530 Subject: [PATCH 17/21] Linting fixes Signed-off-by: Madhav Kandukuri --- tests/unit/mcpgateway/services/test_gateway_service.py | 1 - tests/unit/mcpgateway/test_ui_version.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/mcpgateway/services/test_gateway_service.py b/tests/unit/mcpgateway/services/test_gateway_service.py index 7b7b0c88..69a7fbfa 100644 --- a/tests/unit/mcpgateway/services/test_gateway_service.py +++ b/tests/unit/mcpgateway/services/test_gateway_service.py @@ -278,7 +278,6 @@ async def test_ssl_verification_bypass(self, gateway_service, monkeypatch): Test case logic to verify settings.skip_ssl_verify """ - pass # ──────────────────────────────────────────────────────────────────── # Validate Gateway URL Auth Failure - 401 diff --git a/tests/unit/mcpgateway/test_ui_version.py b/tests/unit/mcpgateway/test_ui_version.py index 1d88cf29..27ddf640 100644 --- a/tests/unit/mcpgateway/test_ui_version.py +++ b/tests/unit/mcpgateway/test_ui_version.py @@ -14,10 +14,12 @@ import base64 from typing import Dict +# Third-Party # Third‑Party import pytest from starlette.testclient import TestClient +# First-Party # First‑Party from mcpgateway.config import settings From 4629a702d49004f34d085960e1058e8e3a27a4cc Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Thu, 31 Jul 2025 14:02:28 +0530 Subject: [PATCH 18/21] Update migration in values.yaml Signed-off-by: Madhav Kandukuri --- charts/mcp-stack/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mcp-stack/values.yaml b/charts/mcp-stack/values.yaml index c1e6aa15..4f9d267c 100644 --- a/charts/mcp-stack/values.yaml +++ b/charts/mcp-stack/values.yaml @@ -261,7 +261,7 @@ migration: # Migration command configuration command: waitForDb: "python3 /app/mcpgateway/utils/db_isready.py --max-tries 30 --interval 2 --timeout 5" - migrate: "alembic upgrade head || echo '⚠️ Migration check failed'" + migrate: "alembic -c mcpgateway/alembic.ini upgrade head || echo '⚠️ Migration check failed'" ######################################################################## # POSTGRES DATABASE From 0fe005584ed54bf4b2f23dca4b3ee61f4f562f5e Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Thu, 31 Jul 2025 14:24:15 +0530 Subject: [PATCH 19/21] Use Containerfile.lite in travis Signed-off-by: Madhav Kandukuri --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index ecbf90eb..d989dbbb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,7 +68,7 @@ jobs: script: | set -e echo "πŸ—οΈ Building container..." - docker build -f Containerfile -t mcpgateway/mcpgateway:latest . + docker build -f Containerfile.lite -t mcpgateway/mcpgateway:latest . echo "πŸš€ Launching container..." docker run -d --name mcpgateway -p 4444:4444 \ @@ -77,9 +77,9 @@ jobs: echo "⏳ Waiting for startup..." sleep 10 - echo "πŸ” Hitting health endpoint..." - curl -fsSL http://localhost:4444/health || { - echo "❌ Health check failed"; docker logs mcpgateway; exit 1; - } + # echo "πŸ” Hitting health endpoint..." + # curl -fsSL http://localhost:4444/health || { + # echo "❌ Health check failed"; docker logs mcpgateway; exit 1; + # } echo "βœ… Container is healthy!" From 420a2cbbd03ecac9f9d9e59d4f6fa6b8fb5037b8 Mon Sep 17 00:00:00 2001 From: Mihai Criveti Date: Sat, 2 Aug 2025 08:06:56 +0100 Subject: [PATCH 20/21] isort flake and cleanup travis/compose adding back healthchecks Signed-off-by: Mihai Criveti --- .github/workflows/snyk.yml.inactive | 26 +++++++++---------- .travis.yml | 8 +++--- docker-compose.yml | 7 +++++ .../versions/9c07b4bc5774_auto_migration.py | 1 + mcpgateway/templates/admin.html | 2 +- .../services/test_gateway_service.py | 2 +- 6 files changed, 27 insertions(+), 19 deletions(-) diff --git a/.github/workflows/snyk.yml.inactive b/.github/workflows/snyk.yml.inactive index e09b994b..c46574e1 100644 --- a/.github/workflows/snyk.yml.inactive +++ b/.github/workflows/snyk.yml.inactive @@ -60,7 +60,7 @@ jobs: dependencies: name: πŸ“¦ Dependency Scan runs-on: ubuntu-latest - + steps: # ------------------------------------------------------------- # 0️⃣ Checkout source @@ -108,7 +108,7 @@ jobs: code-security: name: πŸ” Code Security (SAST) runs-on: ubuntu-latest - + steps: # ------------------------------------------------------------- # 0️⃣ Checkout source @@ -163,7 +163,7 @@ jobs: container-security: name: 🐳 Container Security runs-on: ubuntu-latest - + steps: # ------------------------------------------------------------- # 0️⃣ Checkout source @@ -223,7 +223,7 @@ jobs: iac-security: name: πŸ—οΈ IaC Security runs-on: ubuntu-latest - + steps: # ------------------------------------------------------------- # 0️⃣ Checkout source @@ -253,7 +253,7 @@ jobs: --json-file-output="snyk-iac-${file%.y*ml}.json" || true fi done - + # Test Containerfiles for file in Containerfile*; do if [ -f "$file" ]; then @@ -308,7 +308,7 @@ jobs: name: πŸ“‹ Generate SBOM runs-on: ubuntu-latest if: github.event_name == 'push' && github.ref == 'refs/heads/main' - + steps: # ------------------------------------------------------------- # 0️⃣ Checkout source @@ -337,7 +337,7 @@ jobs: run: | # Get version from pyproject.toml VERSION=$(grep -m1 version pyproject.toml | cut -d'"' -f2 || echo "0.0.0") - + # Generate CycloneDX format snyk sbom \ --format=cyclonedx1.5+json \ @@ -346,7 +346,7 @@ jobs: --version=$VERSION \ --json-file-output=sbom-cyclonedx.json \ . || true - + # Generate SPDX format snyk sbom \ --format=spdx2.3+json \ @@ -376,7 +376,7 @@ jobs: runs-on: ubuntu-latest needs: [dependencies, code-security, container-security, iac-security] if: always() - + steps: # ------------------------------------------------------------- # 0️⃣ Download all artifacts @@ -397,16 +397,16 @@ jobs: echo "**Triggered by:** ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY echo "**Severity Threshold:** ${{ github.event.inputs.severity-threshold || 'high' }}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - + echo "## πŸ“‹ Scan Results" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - + # List all result files echo "### πŸ“ Generated Reports:" >> $GITHUB_STEP_SUMMARY find snyk-results -type f -name "*.json" -o -name "*.sarif" | while read -r file; do echo "- \`$(basename "$file")\`" >> $GITHUB_STEP_SUMMARY done - + echo "" >> $GITHUB_STEP_SUMMARY echo "---" >> $GITHUB_STEP_SUMMARY - echo "*View detailed results in the [Security tab](../../security/code-scanning) or download artifacts from this workflow run.*" >> $GITHUB_STEP_SUMMARY \ No newline at end of file + echo "*View detailed results in the [Security tab](../../security/code-scanning) or download artifacts from this workflow run.*" >> $GITHUB_STEP_SUMMARY diff --git a/.travis.yml b/.travis.yml index d989dbbb..e2ffa94d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -77,9 +77,9 @@ jobs: echo "⏳ Waiting for startup..." sleep 10 - # echo "πŸ” Hitting health endpoint..." - # curl -fsSL http://localhost:4444/health || { - # echo "❌ Health check failed"; docker logs mcpgateway; exit 1; - # } + echo "πŸ” Hitting health endpoint..." + curl -fsSL http://localhost:4444/health || { + echo "❌ Health check failed"; docker logs mcpgateway; exit 1; + } echo "βœ… Container is healthy!" diff --git a/docker-compose.yml b/docker-compose.yml index 2cc9041e..7ecef452 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,6 +63,13 @@ services: migration: condition: service_completed_successfully + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4444/health"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 20s + # volumes: # - ./certs:/app/certs:ro # mount certs folder read-only diff --git a/mcpgateway/alembic/versions/9c07b4bc5774_auto_migration.py b/mcpgateway/alembic/versions/9c07b4bc5774_auto_migration.py index 01f7d5d7..2008eab9 100644 --- a/mcpgateway/alembic/versions/9c07b4bc5774_auto_migration.py +++ b/mcpgateway/alembic/versions/9c07b4bc5774_auto_migration.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """auto migration Revision ID: 9c07b4bc5774 diff --git a/mcpgateway/templates/admin.html b/mcpgateway/templates/admin.html index 69981ff4..0e3db359 100644 --- a/mcpgateway/templates/admin.html +++ b/mcpgateway/templates/admin.html @@ -2710,7 +2710,7 @@

name="template" id="edit-prompt-template" class="mt-1 block w-full h-48 rounded-md border border-gray-300 shadow-sm focus:border-indigo-500 focus:ring-indigo-500" - + >
diff --git a/tests/unit/mcpgateway/services/test_gateway_service.py b/tests/unit/mcpgateway/services/test_gateway_service.py index 69a7fbfa..6fb87618 100644 --- a/tests/unit/mcpgateway/services/test_gateway_service.py +++ b/tests/unit/mcpgateway/services/test_gateway_service.py @@ -432,7 +432,7 @@ async def test_bulk_concurrent_validation(self, gateway_service, monkeypatch): resilient_client_mock.client = mock_client resilient_client_mock.aclose = AsyncMock() - # Patch ResilientHttpClient where it’s used in your module + # Patch ResilientHttpClient where it's used in your module monkeypatch.setattr("mcpgateway.services.gateway_service.ResilientHttpClient", MagicMock(return_value=resilient_client_mock)) # Run the validations concurrently From 4d56d6de97d5be3fcd510f2b46c3a15cbcf7d86a Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Mon, 11 Aug 2025 13:32:59 +0530 Subject: [PATCH 21/21] Remove bootstrap_db.py Signed-off-by: Madhav Kandukuri --- mcpgateway/bootstrap_db.py | 73 -------------------------------------- 1 file changed, 73 deletions(-) delete mode 100644 mcpgateway/bootstrap_db.py diff --git a/mcpgateway/bootstrap_db.py b/mcpgateway/bootstrap_db.py deleted file mode 100644 index 0d7f4761..00000000 --- a/mcpgateway/bootstrap_db.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8 -*- -"""Database bootstrap/upgrade entry-point for MCP Gateway. - -Copyright 2025 -SPDX-License-Identifier: Apache-2.0 -Authors: Madhav Kandukuri - -The script: - -1. Creates a synchronous SQLAlchemy ``Engine`` from ``settings.database_url``. -2. Looks for an *alembic.ini* two levels up from this file to drive migrations. -3. If the database is still empty (no ``gateways`` table), it: - - builds the base schema with ``Base.metadata.create_all()`` - - stamps the migration head so Alembic knows it is up-to-date -4. Otherwise, it applies any outstanding Alembic revisions. -5. Logs a **"Database ready"** message on success. - -It is intended to be invoked via ``python3 -m mcpgateway.bootstrap_db`` or -directly with ``python3 mcpgateway/bootstrap_db.py``. -""" - -# Standard -import asyncio -from importlib.resources import files - -# Third-Party -from alembic import command -from alembic.config import Config -from sqlalchemy import create_engine, inspect - -# First-Party -from mcpgateway.config import settings -from mcpgateway.db import Base -from mcpgateway.services.logging_service import LoggingService - -# Initialize logging service first -logging_service = LoggingService() -logger = logging_service.get_logger(__name__) - - -async def main() -> None: - """ - Bootstrap or upgrade the database schema, then log readiness. - - Runs `create_all()` + `alembic stamp head` on an empty DB, otherwise just - executes `alembic upgrade head`, leaving application data intact. - - Args: - None - """ - engine = create_engine(settings.database_url) - ini_path = files("mcpgateway").joinpath("alembic.ini") - cfg = Config(str(ini_path)) # path in container - cfg.attributes["configure_logger"] = True - - with engine.begin() as conn: - cfg.attributes["connection"] = conn - cfg.set_main_option("sqlalchemy.url", settings.database_url) - - insp = inspect(conn) - - if "gateways" not in insp.get_table_names(): - logger.info("Empty DB detected - creating baseline schema") - Base.metadata.create_all(bind=conn) - command.stamp(cfg, "head") - else: - command.upgrade(cfg, "head") - - logger.info("Database ready") - - -if __name__ == "__main__": - asyncio.run(main())