diff --git a/README.md b/README.md index 57b3495..878e636 100644 --- a/README.md +++ b/README.md @@ -113,11 +113,12 @@ The following environment variables are typically set in `.env`. Docker Compose ### Application -| Environment Variable | Default | Example | Description | -|----------------------------------------|-------------------------|-------------------------------------------|-------------------------------------------------------------------------------------------| -| `MPT_API_BASE_URL` | `http://localhost:8000` | `https://portal.softwareone.com/mpt` | SoftwareONE Marketplace API URL | -| `MPT_API_TOKEN` | - | eyJhbGciOiJSUzI1N... | SoftwareONE Marketplace API Token | -| `MPT_TOOL_STORAGE_TYPE` | `local` | `airtable` | Storage type for MPT tools (local or airtable) | -| `MPT_TOOL_STORAGE_AIRTABLE_API_KEY` | - | patXXXXXXXXXXXXXX | Airtable API key for MPT tool storage (required when storage type is airtable) | -| `MPT_TOOL_STORAGE_AIRTABLE_BASE_ID` | - | appXXXXXXXXXXXXXX | Airtable base ID for MPT tool storage (required when storage type is airtable) | -| `MPT_TOOL_STORAGE_AIRTABLE_TABLE_NAME` | - | MigrationTracking | Airtable table name for MPT tool storage (required when storage type is airtable) | +| Environment Variable | Default | Example | Description | +|----------------------------------------|-------------------------|--------------------------------------|---------------------------------------------------------------------------------------------| +| `MPT_API_BASE_URL` | `http://localhost:8000` | `https://portal.softwareone.com/mpt` | SoftwareONE Marketplace API URL | +| `MPT_API_TOKEN` | - | eyJhbGciOiJSUzI1N... | SoftwareONE Marketplace API Token | +| `MPT_TOOL_STORAGE_TYPE` | `local` | `airtable` | Storage type for MPT tools (local or airtable) | +| `MPT_TOOL_STORAGE_AIRTABLE_API_KEY` | - | patXXXXXXXXXXXXXX | Airtable API key for MPT tool storage (required when storage type is airtable) | +| `MPT_TOOL_STORAGE_AIRTABLE_BASE_ID` | - | appXXXXXXXXXXXXXX | Airtable base ID for MPT tool storage (required when storage type is airtable) | +| `MPT_TOOL_STORAGE_AIRTABLE_TABLE_NAME` | - | MigrationTracking | Airtable table name for MPT tool storage (required when storage type is airtable) | +| `SERVICE_VERSION` | empty | `5.4.2` | Optional service version saved in migration state when a migration state is created | diff --git a/docs/PROJECT_DESCRIPTION.md b/docs/PROJECT_DESCRIPTION.md index ca9c5c6..c68aff8 100644 --- a/docs/PROJECT_DESCRIPTION.md +++ b/docs/PROJECT_DESCRIPTION.md @@ -46,6 +46,7 @@ The tool uses the following environment variables: - `MPT_API_TOKEN`: Your MPT API key (required when using `MPTAPIClientMixin`) - `MPT_TOOL_STORAGE_TYPE`: Storage backend for migration state (`local` or `airtable`, default: `local`). See [Storage Configuration](#storage) - `MPT_TOOL_STORAGE_AIRTABLE_API_KEY`: Your Airtable API key (required when using `AirtableAPIClientMixin` or when `MPT_TOOL_STORAGE_TYPE=airtable`) +- `SERVICE_VERSION`: Optional service version persisted into each new migration state. If missing, the stored value is empty ## Configuration @@ -77,6 +78,7 @@ Your Airtable table must have the following columns: | started_at | dateTime | ❌ | | applied_at | dateTime | ❌ | | type | singleSelect (data, schema) | ✅ | +| version | singleLineText | ❌ | **Airtable configuration steps:** @@ -220,23 +222,25 @@ When running a single migration (`--data MIGRATION_ID` or `--schema MIGRATION_ID "order_id": 20260113180013, "started_at": "2026-01-13T18:05:20.000000", "applied_at": "2026-01-13T18:05:23.123456", - "type": "data" + "type": "data", + "version": "5.3.2" }, "schema_example": { "migration_id": "schema_example", "order_id": 20260214121033, "started_at": null, "applied_at": null, - "type": "schema" + "type": "schema", + "version": "" } } ``` **Migration Table (Airtable):** -| order_id | migration_id | started_at | applied_at | type | -|----------------|----------------|----------------------------|----------------------------|--------| -| 20260113180013 | data_example | 2026-01-13T18:05:20.000000 | 2026-01-13T18:05:23.123456 | data | -| 20260214121033 | schema_example | | | schema | +| order_id | migration_id | started_at | applied_at | type | version | +|----------------|----------------|----------------------------|----------------------------|--------|---------| +| 20260113180013 | data_example | 2026-01-13T18:05:20.000000 | 2026-01-13T18:05:23.123456 | data | 5.3.2 | +| 20260214121033 | schema_example | | | schema | | If a migration succeeds during execution: @@ -279,6 +283,9 @@ To see all migrations and their status: ``` The output shows execution order, status, and timestamps. +It also shows the `version` column as the last column: +- Applied/created state: value from `SERVICE_VERSION`, or empty if the variable is not set +- Not applied (no state yet): `-` The status column is derived from the persisted timestamps: diff --git a/mpt_tool/config.py b/mpt_tool/config.py index 3fe8642..b57882b 100644 --- a/mpt_tool/config.py +++ b/mpt_tool/config.py @@ -17,6 +17,11 @@ def get_mpt_config(config_key: str) -> str | None: return config.get(config_key) +def get_service_version() -> str | None: + """Get service version.""" + return os.getenv("SERVICE_VERSION") + + def get_storage_type() -> str: """Get storage type.""" return os.getenv("MPT_TOOL_STORAGE_TYPE", "local") diff --git a/mpt_tool/managers/state/airtable.py b/mpt_tool/managers/state/airtable.py index cf27359..08f955e 100644 --- a/mpt_tool/managers/state/airtable.py +++ b/mpt_tool/managers/state/airtable.py @@ -5,7 +5,7 @@ from pyairtable.orm import Model, fields from requests import HTTPError -from mpt_tool.config import get_airtable_config +from mpt_tool.config import get_airtable_config, get_service_version from mpt_tool.enums import MigrationTypeEnum from mpt_tool.managers import StateManager from mpt_tool.managers.errors import InitializationError, StateNotFoundError @@ -20,6 +20,7 @@ class MigrationStateModel(Model): type = fields.RequiredSelectField("type") started_at = fields.DatetimeField("started_at") applied_at = fields.DatetimeField("applied_at") + version = fields.TextField("version") class Meta: @staticmethod @@ -64,6 +65,7 @@ def get_by_id(cls, migration_id: str) -> Migration: type=MigrationTypeEnum(state.type), started_at=state.started_at, applied_at=state.applied_at, + version=state.version, ) @override @@ -105,6 +107,7 @@ def initialize(cls) -> None: "timeZone": "utc", }, }, + {"name": "version", "type": "singleLineText"}, ] try: base.create_table(table_name, fields=table_fields) @@ -122,6 +125,7 @@ def load(cls) -> dict[str, Migration]: type=MigrationTypeEnum(state.type), started_at=state.started_at, applied_at=state.applied_at, + version=state.version, ) migrations[migration.migration_id] = migration @@ -131,7 +135,10 @@ def load(cls) -> dict[str, Migration]: @classmethod def new(cls, migration_id: str, migration_type: MigrationTypeEnum, order_id: int) -> Migration: state = MigrationStateModel( - migration_id=migration_id, order_id=order_id, type=migration_type.value + migration_id=migration_id, + order_id=order_id, + type=migration_type.value, + version=get_service_version(), ) state.save() return Migration( @@ -140,6 +147,7 @@ def new(cls, migration_id: str, migration_type: MigrationTypeEnum, order_id: int type=MigrationTypeEnum(state.type), started_at=state.started_at, applied_at=state.applied_at, + version=state.version, ) @override @@ -153,6 +161,7 @@ def save_state(cls, state: Migration) -> None: migration_state_model.type = state.type.value migration_state_model.started_at = state.started_at migration_state_model.applied_at = state.applied_at + migration_state_model.version = state.version else: migration_state_model = MigrationStateModel( migration_id=state.migration_id, @@ -160,6 +169,7 @@ def save_state(cls, state: Migration) -> None: type=state.type.value, started_at=state.started_at, applied_at=state.applied_at, + version=state.version, ) migration_state_model.save() diff --git a/mpt_tool/managers/state/file.py b/mpt_tool/managers/state/file.py index 7322c16..cc59219 100644 --- a/mpt_tool/managers/state/file.py +++ b/mpt_tool/managers/state/file.py @@ -2,6 +2,7 @@ from pathlib import Path from typing import override +from mpt_tool.config import get_service_version from mpt_tool.constants import MIGRATION_STATE_FILE from mpt_tool.enums import MigrationTypeEnum from mpt_tool.managers import StateManager @@ -61,6 +62,7 @@ def new(cls, migration_id: str, migration_type: MigrationTypeEnum, order_id: int migration_id=migration_id, order_id=order_id, type=migration_type, + version=get_service_version(), ) cls.save_state(new_state) return new_state diff --git a/mpt_tool/models.py b/mpt_tool/models.py index e0a04e5..0231b7b 100644 --- a/mpt_tool/models.py +++ b/mpt_tool/models.py @@ -14,8 +14,10 @@ class Migration: migration_id: str order_id: int type: MigrationTypeEnum - started_at: dt.datetime | None = None + applied_at: dt.datetime | None = None + started_at: dt.datetime | None = None + version: str | None = None @classmethod def from_dict(cls, migration_data: dict[str, Any]) -> Self: @@ -24,12 +26,13 @@ def from_dict(cls, migration_data: dict[str, Any]) -> Self: migration_id=migration_data["migration_id"], order_id=migration_data["order_id"], type=MigrationTypeEnum(migration_data["type"]), - started_at=dt.datetime.fromisoformat(migration_data["started_at"]) - if migration_data["started_at"] - else None, applied_at=dt.datetime.fromisoformat(migration_data["applied_at"]) if migration_data["applied_at"] else None, + started_at=dt.datetime.fromisoformat(migration_data["started_at"]) + if migration_data["started_at"] + else None, + version=migration_data.get("version"), ) def to_dict(self) -> dict[str, Any]: @@ -38,8 +41,9 @@ def to_dict(self) -> dict[str, Any]: "migration_id": self.migration_id, "order_id": self.order_id, "type": self.type.value, - "started_at": self.started_at.isoformat() if self.started_at else None, "applied_at": self.applied_at.isoformat() if self.applied_at else None, + "started_at": self.started_at.isoformat() if self.started_at else None, + "version": self.version, } def applied(self) -> None: @@ -48,8 +52,9 @@ def applied(self) -> None: def failed(self) -> None: """Mark the migration as failed.""" - self.started_at = None self.applied_at = None + self.started_at = None + self.version = None def manual(self) -> None: """Mark the migration as manual.""" @@ -118,11 +123,13 @@ class MigrationListItem: migration_id: str order_id: int - migration_type: MigrationTypeEnum | None - started_at: dt.datetime | None - applied_at: dt.datetime | None status: MigrationStatusEnum + applied_at: dt.datetime | None = None + migration_type: MigrationTypeEnum | None = None + started_at: dt.datetime | None = None + version: str | None = None + @classmethod def from_sources( cls, @@ -133,8 +140,9 @@ def from_sources( return cls( migration_id=migration_file.migration_id, order_id=migration_file.order_id, + status=MigrationStatusEnum.from_state(migration_state), + applied_at=migration_state.applied_at if migration_state else None, migration_type=migration_state.type if migration_state else None, started_at=migration_state.started_at if migration_state else None, - applied_at=migration_state.applied_at if migration_state else None, - status=MigrationStatusEnum.from_state(migration_state), + version=migration_state.version if migration_state else None, ) diff --git a/mpt_tool/renders.py b/mpt_tool/renders.py index f7c43c4..db4c1f2 100644 --- a/mpt_tool/renders.py +++ b/mpt_tool/renders.py @@ -7,7 +7,7 @@ class MigrationRender: """Render migration state information as a formatted table.""" - _fields = ("order_id", "migration_id", "started_at", "applied_at", "type", "status") + _fields = ("order_id", "migration_id", "started_at", "applied_at", "type", "status", "version") def __init__(self, migration_items: list[MigrationListItem]): self.migration_items = migration_items @@ -25,6 +25,7 @@ def __rich_console__(self, console: Console, options: ConsoleOptions) -> RenderR migration.applied_at.isoformat(timespec="seconds") if migration.applied_at else "-", migration.migration_type.value if migration.migration_type else "-", migration.status.title(), + migration.version or "-", ) yield table diff --git a/tests/cli/test_cli_airtable_storage.py b/tests/cli/test_cli_airtable_storage.py index a72e89a..433f24d 100644 --- a/tests/cli/test_cli_airtable_storage.py +++ b/tests/cli/test_cli_airtable_storage.py @@ -33,6 +33,7 @@ def applied_migration(data_migration_file, mock_airtable): "type": "data", "started_at": "2025-04-06T13:00:00+00:00", "applied_at": "2025-04-06T13:00:00+00:00", + "version": None, } ], ) @@ -54,6 +55,7 @@ def test_migrate_data_migration(mock_airtable, runner, log): "type": "data", "started_at": "2025-04-06T13:00:10.000Z", "applied_at": "2025-04-06T13:00:10.000Z", + "version": "", } assert "Running data migrations..." in result.output assert "Running migration: fake_data_file_name" in log.text @@ -73,7 +75,9 @@ def test_migrate_skip_migration_already_applied(runner, log): @freeze_time("2025-04-06 13:00:00") @pytest.mark.usefixtures("data_migration_file_error") -def test_migrate_data_run_script_fail(mock_airtable, runner, log): +def test_migrate_data_run_script_fail(monkeypatch, mock_airtable, runner, log): + monkeypatch.setenv("SERVICE_VERSION", "5.1.2") + result = runner.invoke(app, ["migrate", "--data"]) assert result.exit_code == 1, result.output @@ -85,6 +89,7 @@ def test_migrate_data_run_script_fail(mock_airtable, runner, log): "type": "data", "started_at": None, "applied_at": None, + "version": None, } assert "Running data migrations..." in result.output assert "Running migration: fake_error_file_name" in log.text @@ -93,7 +98,9 @@ def test_migrate_data_run_script_fail(mock_airtable, runner, log): @freeze_time("2025-04-06 10:11:24") @pytest.mark.usefixtures("data_migration_file") -def test_migrate_manual(mock_airtable, runner): +def test_migrate_manual(monkeypatch, mock_airtable, runner): + monkeypatch.setenv("SERVICE_VERSION", "5.2.3") + result = runner.invoke(app, ["migrate", "--manual", "fake_data_file_name"]) assert result.exit_code == 0, result.output @@ -107,6 +114,7 @@ def test_migrate_manual(mock_airtable, runner): "type": "data", "started_at": None, "applied_at": "2025-04-06T10:11:24.000Z", + "version": "5.2.3", } @@ -158,6 +166,7 @@ def test_migrate_init(mocker, runner): "timeZone": "utc", }, }, + {"name": "version", "type": "singleLineText"}, ], ) @@ -182,9 +191,9 @@ def test_migrate_list(runner, log): assert result.exit_code == 0, result.output assert "No state found for migration: fake_schema_file_name" in log.text formatted_output = "".join(result.output.split()) - assert "┃order_id┃migration_id┃started_at┃applied_at┃type┃status┃" in formatted_output + assert "┃order_id┃migration_id┃started_at┃applied_at┃type┃status┃version┃" in formatted_output assert ( - "│20250406020202│fake_data_file_name│2025-04-06T13:00:00+00:00│2025-04-06T13:00:00+00:00│data│Applied│" + "│20250406020202│fake_data_file_name│2025-04-06T13:00:00+00:00│2025-04-06T13:00:00+00:00│data│Applied│-│" in formatted_output ) - assert "│20260101010101│fake_schema_file_name│-│-│-│NotApplied│" in formatted_output + assert "│20260101010101│fake_schema_file_name│-│-│-│NotApplied│-│" in formatted_output diff --git a/tests/cli/test_cli_local_storage.py b/tests/cli/test_cli_local_storage.py index 7722ec9..718de49 100644 --- a/tests/cli/test_cli_local_storage.py +++ b/tests/cli/test_cli_local_storage.py @@ -23,6 +23,7 @@ def applied_migration(data_migration_file, migration_state_file): "type": "data", "started_at": "2025-04-06T13:10:20+00:00", "applied_at": "2025-04-06T13:10:30+00:00", + "version": "5.3.2", } } migration_state_file.write_text(encoding="utf-8", data=json.dumps(applied_state_data)) @@ -32,7 +33,9 @@ def applied_migration(data_migration_file, migration_state_file): @freeze_time("2025-04-06 13:10:30") @pytest.mark.usefixtures("data_migration_file", "schema_migration_file") -def test_migrate_data_migration(migration_state_file, runner, log): +def test_migrate_data_migration(monkeypatch, migration_state_file, runner, log): + monkeypatch.setenv("SERVICE_VERSION", "1.2.3") + result = runner.invoke(app, ["migrate", "--data"]) assert result.exit_code == 0, result.output @@ -44,6 +47,7 @@ def test_migrate_data_migration(migration_state_file, runner, log): "type": "data", "started_at": "2025-04-06T13:10:30+00:00", "applied_at": "2025-04-06T13:10:30+00:00", + "version": "1.2.3", } } assert "Running data migrations..." in result.output @@ -65,6 +69,7 @@ def test_migrate_data_single_migration(migration_state_file, runner, log): "type": "data", "started_at": "2025-04-06T13:10:30+00:00", "applied_at": "2025-04-06T13:10:30+00:00", + "version": None, } } assert "Running data migrations..." in result.output @@ -86,6 +91,7 @@ def test_migrate_skip_migration_already_applied(migration_state_file, runner, lo "type": "data", "started_at": "2025-04-06T13:10:20+00:00", "applied_at": "2025-04-06T13:10:30+00:00", + "version": "5.3.2", } } assert "Running data migrations..." in result.output @@ -115,6 +121,7 @@ def test_migrate_data_run_script_fail(migration_state_file, runner, log): "type": "data", "started_at": None, "applied_at": None, + "version": None, } assert "Running data migrations..." in result.output assert "Running migration: fake_error_file_name" in log.text @@ -154,6 +161,7 @@ def test_migrate_schema_single_migration(migration_state_file, runner, log): "type": "schema", "started_at": "2025-04-06T13:10:30+00:00", "applied_at": "2025-04-06T13:10:30+00:00", + "version": None, } } assert "Running schema migrations..." in result.output @@ -177,6 +185,7 @@ def test_migrate_manual(migration_state_file, runner): "type": "data", "started_at": None, "applied_at": "2025-04-06T10:11:24+00:00", + "version": None, } } @@ -219,9 +228,9 @@ def test_migrate_list(runner, log): assert result.exit_code == 0, result.output assert "No state found for migration: fake_schema_file_name" in log.text formatted_output = "".join(result.output.split()) - assert "┃order_id┃migration_id┃started_at┃applied_at┃type┃status┃" in formatted_output + assert "┃order_id┃migration_id┃started_at┃applied_at┃type┃status┃version┃" in formatted_output assert ( - "│20250406020202│fake_data_file_name│2025-04-06T13:10:20+00:00│2025-04-06T13:10:30+00:00│data│Applied│" + "│20250406020202│fake_data_file_name│2025-04-06T13:10:20+00:00│2025-04-06T13:10:30+00:00│data│Applied│5.3.2│" in formatted_output ) - assert "│20260101010101│fake_schema_file_name│-│-│-│NotApplied│" in formatted_output + assert "│20260101010101│fake_schema_file_name│-│-│-│NotApplied│-│" in formatted_output