added version checks for manifest migrations. Added auto-import for all migrations available

Author: Oleksandr Bazarnov

airbyte_cdk/sources/declarative/migrations/manifest/manifest_migration.py

@@ -1,5 +1,6 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 
+import re
 from abc import abstractmethod
 from typing import Any, Dict
 
@@ -34,6 +35,15 @@ def migrate(self, manifest: ManifestType) -> None:
         :param kwargs: Additional arguments for migration
         """
 
+    @property
+    def migration_version(self) -> str:
+        """
+        Get the migration version.
+
+        :return: The migration version as a string
+        """
+        return self._get_migration_version()
+
     def _is_component(self, obj: Dict[str, Any]) -> bool:
         """
         Check if the object is a component.
@@ -45,12 +55,16 @@ def _is_component(self, obj: Dict[str, Any]) -> bool:
 
     def _is_migratable(self, obj: Dict[str, Any]) -> bool:
         """
-        Check if the object is a migratable component.
+        Check if the object is a migratable component,
+        based on the Type of the component and the migration version.
 
         :param obj: The object to check
         :return: True if the object is a migratable component, False otherwise
         """
-        return obj[TYPE_TAG] not in NON_MIGRATABLE_TYPES
+        return (
+            obj[TYPE_TAG] not in NON_MIGRATABLE_TYPES
+            and self._get_manifest_version(obj) <= self.migration_version
+        )
 
     def _process_manifest(self, obj: Any) -> None:
         """
@@ -91,3 +105,33 @@ def _process_manifest(self, obj: Any) -> None:
             # Process all items in the list
             for item in obj:
                 self._process_manifest(item)
+
+    def _get_manifest_version(self, manifest: ManifestType) -> str:
+        """
+        Get the manifest version from the manifest.
+
+        :param manifest: The manifest to get the version from
+        :return: The manifest version
+        """
+        return manifest.get("version", "0.0.0")
+
+    def _get_migration_version(self) -> str:
+        """
+        Get the migration version from the class name.
+        The migration version is extracted from the class name using a regular expression.
+        The expected format is "V_<major>_<minor>_<patch>_<migration_name>".
+
+        For example, "V_6_45_2_ManifestMigration_HttpRequesterPathToUrl" -> "6.45.2"
+
+        :return: The migration version as a string in the format "major.minor.patch"
+        :raises ValueError: If the class name does not match the expected format
+        """
+
+        class_name = self.__class__.__name__
+        migration_version = re.search(r"V_(\d+_\d+_\d+)", class_name)
+        if migration_version:
+            return migration_version.group(1).replace("_", ".")
+        else:
+            raise ValueError(
+                f"Invalid migration class name, make sure the class name has the version (e.g `V_0_0_0_`): {class_name}"
+            )

airbyte_cdk/sources/declarative/migrations/manifest/migration_handler.py

@@ -13,7 +13,7 @@
     ManifestType,
 )
 from airbyte_cdk.sources.declarative.migrations.manifest.migrations_registry import (
-    migrations_registry,
+    MIGRATIONS,
 )
 
 
@@ -39,8 +39,8 @@ def apply_migrations(self) -> ManifestType:
                           manifest if any migration failed.
         """
         try:
-            for migration_class in migrations_registry:
-                self._handle_migration(migration_class)
+            for migration_cls in MIGRATIONS:
+                self._handle_migration(migration_cls)
             return self._migrated_manifest
         except ManifestMigrationException:
             # if any errors occur we return the original resolved manifest

…p_requester_url_base_to_url_migration.py …p_requester_url_base_to_url_migration.pyairbyte_cdk/sources/declarative/migrations/manifest/migrations/http_requester_url_base_to_url_migration.py renamed to airbyte_cdk/sources/declarative/migrations/manifest/migrations/0_v6_45_2_http_requester_url_base_to_url_migration.py airbyte_cdk/sources/declarative/migrations/manifest/migrations/http_requester_url_base_to_url_migration.py renamed to airbyte_cdk/sources/declarative/migrations/manifest/migrations/0_v6_45_2_http_requester_url_base_to_url_migration.py

@@ -5,7 +5,7 @@
 )
 
 
-class HttpRequesterUrlBaseToUrlMigration(ManifestMigration):
+class V_6_45_2_ManifestMigration_HttpRequesterUrlBaseToUrl(ManifestMigration):
     """
     This migration is responsible for migrating the `url_base` key to `url` in the HttpRequester component.
     The `url_base` key is expected to be a base URL, and the `url` key is expected to be a full URL.

…/http_requester_path_to_url_migration.py …_http_requester_path_to_url_migration.pyairbyte_cdk/sources/declarative/migrations/manifest/migrations/http_requester_path_to_url_migration.py renamed to airbyte_cdk/sources/declarative/migrations/manifest/migrations/1_v6_45_2_http_requester_path_to_url_migration.py airbyte_cdk/sources/declarative/migrations/manifest/migrations/http_requester_path_to_url_migration.py renamed to airbyte_cdk/sources/declarative/migrations/manifest/migrations/1_v6_45_2_http_requester_path_to_url_migration.py

@@ -8,7 +8,7 @@
 from airbyte_cdk.sources.types import EmptyString
 
 
-class HttpRequesterPathToUrlMigration(ManifestMigration):
+class V_6_45_2_ManifestMigration_HttpRequesterPathToUrl(ManifestMigration):
     """
     This migration is responsible for migrating the `path` key to `url` in the HttpRequester component.
     The `path` key is expected to be a relative path, and the `url` key is expected to be a full URL.

airbyte_cdk/sources/declarative/migrations/manifest/migrations/__init__.py

@@ -1,11 +0,0 @@
-from airbyte_cdk.sources.declarative.migrations.manifest.migrations.http_requester_path_to_url_migration import (
-    HttpRequesterPathToUrlMigration,
-)
-from airbyte_cdk.sources.declarative.migrations.manifest.migrations.http_requester_url_base_to_url_migration import (
-    HttpRequesterUrlBaseToUrlMigration,
-)
-
-__all__ = [
-    "HttpRequesterUrlBaseToUrlMigration",
-    "HttpRequesterPathToUrlMigration",
-]

airbyte_cdk/sources/declarative/migrations/manifest/migrations_registry.py

@@ -1,17 +1,53 @@
+import importlib
+import inspect
+import pkgutil
+import re
+import sys
 from typing import List, Type
 
+import airbyte_cdk.sources.declarative.migrations.manifest.migrations as migrations_pkg
 from airbyte_cdk.sources.declarative.migrations.manifest.manifest_migration import (
     ManifestMigration,
 )
-from airbyte_cdk.sources.declarative.migrations.manifest.migrations import (
-    HttpRequesterPathToUrlMigration,
-    HttpRequesterUrlBaseToUrlMigration,
-)
 
-# This is the registry of all the migrations that are available.
-# Add new migrations to the bottom of the list,
-# ( ! ) make sure the order of the migrations is correct.
-migrations_registry: List[Type[ManifestMigration]] = [
-    HttpRequesterUrlBaseToUrlMigration,
-    HttpRequesterPathToUrlMigration,
-]
+# Dynamically import all modules in the migrations package
+for _, module_name, is_pkg in pkgutil.iter_modules(migrations_pkg.__path__):
+    if not is_pkg:
+        importlib.import_module(f"{migrations_pkg.__name__}.{module_name}")
+
+
+def _migration_order_key(cls):
+    # Extract the migration order from the module name, e.g., 0_v6_45_2_http_requester_url_base_to_url_migration
+    # The order is the integer at the start of the module name, before the first underscore
+    module_name = cls.__module__.split(".")[-1]
+    match = re.match(r"(\d+)_", module_name)
+    return int(match.group(1)) if match else float("inf")
+
+
+def _discover_migrations() -> List[Type[ManifestMigration]]:
+    migration_classes = []
+    for name, obj in inspect.getmembers(sys.modules[migrations_pkg.__name__], inspect.isclass):
+        if (
+            issubclass(obj, ManifestMigration)
+            and obj is not ManifestMigration
+            and obj not in migration_classes
+        ):
+            migration_classes.append(obj)
+
+    for _, module_name, _ in pkgutil.iter_modules(migrations_pkg.__path__):
+        module = sys.modules.get(f"{migrations_pkg.__name__}.{module_name}")
+        if module:
+            for name, obj in inspect.getmembers(module, inspect.isclass):
+                if (
+                    issubclass(obj, ManifestMigration)
+                    and obj is not ManifestMigration
+                    and obj not in migration_classes
+                ):
+                    migration_classes.append(obj)
+
+    # Sort by migration order key
+    migration_classes.sort(key=_migration_order_key)
+    return migration_classes
+
+
+MIGRATIONS: List[Type[ManifestMigration]] = _discover_migrations()