fixed model registries and model export

Author: eadwinCode

docs/migrations/env.md

@@ -138,7 +138,7 @@ class MyCustomMigrationEnv(AlembicEnvMigrationBase):
 
         """
         key, engine = self.db_service.engines.popitem()
-        metadata = get_metadata(key, certain=True)
+        metadata = get_metadata(key, certain=True).metadata
 
         conf_args = {}
         conf_args.setdefault(

docs/testing/index.md

@@ -0,0 +1,10 @@
+
+
+
+## Testing Fixtures
+
+## Alembic Migration with Test Fixture
+
+## Testing a model
+
+## Factory Boy

ellar_sql/exceptions.py

@@ -104,7 +104,7 @@ def run_migrations_offline(
             logger.info("Migrating database %s" % key)
 
             url = str(engine.url).replace("%", "%%")
-            metadata = get_metadata(key, certain=True)
+            metadata = get_metadata(key, certain=True).metadata
 
             file_ = "%s.sql" % key
             logger.info("Writing output to %s" % file_)
@@ -170,7 +170,7 @@ async def _compute_engine_info(self) -> t.List[DatabaseInfo]:
         res = []
 
         for key, engine in self.db_service.engines.items():
-            metadata = get_metadata(key, certain=True)
+            metadata = get_metadata(key, certain=True).metadata
 
             if engine.dialect.is_async:
                 async_engine = AsyncEngine(engine)

ellar_sql/migrations/multiple.py

@@ -53,7 +53,7 @@ def run_migrations_offline(
         """
 
         key, engine = self.db_service.engines.popitem()
-        metadata = get_metadata(key, certain=True)
+        metadata = get_metadata(key, certain=True).metadata
 
         conf_args = self.get_user_context_configurations()
 
@@ -95,7 +95,7 @@ async def run_migrations_online(self, context: "EnvironmentContext") -> None:
         """
 
         key, engine = self.db_service.engines.popitem()
-        metadata = get_metadata(key, certain=True)
+        metadata = get_metadata(key, certain=True).metadata
 
         migration_action_partial = functools.partial(
             self._migration_action, metadata=metadata, context=context

ellar_sql/migrations/single.py

@@ -26,7 +26,9 @@ def _update_metadata(namespace: t.Dict[str, t.Any]) -> None:
         if not has_metadata(database_key):
             # verify the key exist and store the metadata
             metadata.info[DATABASE_BIND_KEY] = database_key
-            update_database_metadata(database_key, metadata)
+            update_database_metadata(
+                database_key, metadata, sa_orm.registry(metadata=metadata)
+            )
         # if we have saved metadata then, its save to remove it and allow
         # DatabaseBindKeyMixin to set it back when the class if fully created.
         namespace.pop("metadata")
@@ -107,29 +109,30 @@ def __new__(
                 lambda ns: ns.update(namespace),
             )
 
+            # model = t.cast(t.Type[sa_orm.DeclarativeBase], model)
+
             if not has_metadata(DEFAULT_KEY):
                 # Use the model's metadata as the default metadata.
                 model.metadata.info[DATABASE_BIND_KEY] = DEFAULT_KEY
-                update_database_metadata(DEFAULT_KEY, model.metadata)
+                update_database_metadata(DEFAULT_KEY, model.metadata, model.registry)
             elif not has_metadata(model.metadata.info.get(DATABASE_BIND_KEY)):
                 # Use the passed in default metadata as the model's metadata.
                 model.metadata = get_metadata(DEFAULT_KEY, certain=True)
 
             return model
 
         # _update_metadata(namespace)
+        __base_config__ = ModelBaseConfig(use_bases=options.use_bases, as_base=True)
 
         base = ModelMeta(
             "ModelBase",
             bases,
-            {
-                "__base_config__": ModelBaseConfig(
-                    use_bases=options.use_bases, as_base=True
-                )
-            },
+            {"__base_config__": __base_config__},
         )
 
-        return types.new_class(name, (base,), {}, lambda ns: ns.update(namespace))
+        return types.new_class(
+            name, (base,), {"options": options}, lambda ns: ns.update(namespace)
+        )
 
 
 class ModelBase(ModelDataExportMixin):
@@ -157,9 +160,6 @@ class ModelBase(ModelDataExportMixin):
         def __init__(self, **kwargs: t.Any) -> None:
             ...
 
-        def dict(self, exclude: t.Optional[t.Set[str]] = None) -> t.Dict[str, t.Any]:
-            ...
-
         def _sa_inspect_type(self) -> sa.Mapper["ModelBase"]:
             ...
 

ellar_sql/model/base.py

@@ -1,27 +1,40 @@
 import typing as t
 
 import sqlalchemy as sa
-
-__model_database_metadata__: t.Dict[str, sa.MetaData] = {}
+import sqlalchemy.orm as sa_orm
 
 from ellar_sql.constant import DEFAULT_KEY
 
 
-def update_database_metadata(database_key: str, value: sa.MetaData) -> None:
+class DatabaseMetadata(t.NamedTuple):
+    metadata: sa.MetaData
+    registry: sa_orm.registry
+
+
+__model_database_metadata__: t.Dict[str, DatabaseMetadata] = {}
+
+
+def update_database_metadata(
+    database_key: str, value: sa.MetaData, registry: sa_orm.registry
+) -> None:
     """
     Update a metadata based on a database key
     """
-    __model_database_metadata__[database_key] = value
+    __model_database_metadata__[database_key] = DatabaseMetadata(
+        metadata=value, registry=registry
+    )
 
 
-def get_all_metadata() -> t.Dict[str, sa.MetaData]:
+def get_all_metadata() -> t.Dict[str, DatabaseMetadata]:
     """
     Get all metadata available in your application
     """
     return __model_database_metadata__.copy()
 
 
-def get_metadata(database_key: str = DEFAULT_KEY, certain: bool = False) -> sa.MetaData:
+def get_metadata(
+    database_key: str = DEFAULT_KEY, certain: bool = False
+) -> DatabaseMetadata:
     """
     Gets Metadata associated with a database key
     """

ellar_sql/model/database_binds.py

@@ -1,10 +1,23 @@
 import typing as t
 
 import sqlalchemy as sa
+import sqlalchemy.orm as sa_orm
+from pydantic.v1 import BaseModel
 
 from ellar_sql.constant import ABSTRACT_KEY, DATABASE_KEY, DEFAULT_KEY, TABLE_KEY
+from ellar_sql.model.utils import (
+    camel_to_snake_case,
+    make_metadata,
+    should_set_table_name,
+)
+from ellar_sql.schemas import ModelBaseConfig, ModelMetaStore
 
-from .utils import camel_to_snake_case, make_metadata, should_set_table_name
+
+class asss(BaseModel):
+    sd: str
+
+
+IncEx = t.Union[t.Set[int], t.Set[str], t.Dict[int, t.Any], t.Dict[str, t.Any]]
 
 if t.TYPE_CHECKING:
     from .base import ModelBase
@@ -30,33 +43,48 @@ def __init_subclass__(cls, **kwargs: t.Dict[str, t.Any]) -> None:
 
 class DatabaseBindKeyMixin:
     metadata: sa.MetaData
-    __dnd__ = "Ellar"
 
     def __init_subclass__(cls, **kwargs: t.Dict[str, t.Any]) -> None:
         if not ("metadata" in cls.__dict__ or TABLE_KEY in cls.__dict__) and hasattr(
             cls, DATABASE_KEY
         ):
             database_bind_key = getattr(cls, DATABASE_KEY, DEFAULT_KEY)
             parent_metadata = getattr(cls, "metadata", None)
-            metadata = make_metadata(database_bind_key)
+            db_metadata = make_metadata(database_bind_key)
 
-            if metadata is not parent_metadata:
-                cls.metadata = metadata
+            if db_metadata.metadata is not parent_metadata:
+                cls.metadata = db_metadata.metadata
+                cls.registry = db_metadata.registry  # type:ignore[attr-defined]
 
         super().__init_subclass__(**kwargs)
 
 
 class ModelTrackMixin:
     metadata: sa.MetaData
+    __mms__: ModelMetaStore
+    __table__: sa.Table
 
     def __init_subclass__(cls, **kwargs: t.Dict[str, t.Any]) -> None:
+        options: ModelBaseConfig = kwargs.pop(  # type:ignore[assignment]
+            "options",
+            ModelBaseConfig(as_base=False, use_bases=[sa_orm.DeclarativeBase]),
+        )
+
         super().__init_subclass__(**kwargs)
 
         if TABLE_KEY in cls.__dict__ and ABSTRACT_KEY not in cls.__dict__:
             __ellar_sqlalchemy_models__[str(cls)] = cls  # type:ignore[assignment]
 
+            cls.__mms__ = ModelMetaStore(
+                base_config=options,
+                pk_column=None,
+                columns=list(cls.__table__.columns),  # type:ignore[arg-type]
+            )
+
 
 class ModelDataExportMixin:
+    __mms__: t.Optional[ModelMetaStore] = None
+
     def __repr__(self) -> str:
         state = sa.inspect(self)
         assert state is not None
@@ -70,13 +98,46 @@ def __repr__(self) -> str:
 
         return f"<{type(self).__name__} {pk}>"
 
-    def dict(self, exclude: t.Optional[t.Set[str]] = None) -> t.Dict[str, t.Any]:
+    def _calculate_keys(
+        self,
+        include: t.Optional[t.Set[str]],
+        exclude: t.Optional[t.Set[str]],
+    ) -> t.Set[str]:
+        keys: t.Set[str] = {k for k in self.__dict__.keys() if not k.startswith("_sa")}
+
+        if include is None and exclude is None:
+            return keys
+
+        if include is not None:
+            keys &= include
+
+        if exclude:
+            keys -= exclude
+
+        return keys
+
+    def _iter(
+        self,
+        include: t.Optional[t.Set[str]],
+        exclude: t.Optional[t.Set[str]],
+        exclude_none: bool = False,
+    ) -> t.Generator[t.Tuple[str, t.Any], None, None]:
+        allowed_keys = self._calculate_keys(include=include, exclude=exclude)
+
+        for field_key, v in self.__dict__.items():
+            if (allowed_keys is not None and field_key not in allowed_keys) or (
+                exclude_none and v is None
+            ):
+                continue
+            yield field_key, v
+
+    def dict(
+        self,
+        include: t.Optional[t.Set[str]] = None,
+        exclude: t.Optional[t.Set[str]] = None,
+        exclude_none: bool = False,
+    ) -> t.Dict[str, t.Any]:
         # TODO: implement advance exclude and include that goes deep into relationships too
-        _exclude: t.Set[str] = set() if not exclude else exclude
-
-        tuple_generator = (
-            (k, v)
-            for k, v in self.__dict__.items()
-            if k not in _exclude and not k.startswith("_sa")
+        return dict(
+            self._iter(include=include, exclude_none=exclude_none, exclude=exclude)
         )
-        return dict(tuple_generator)

ellar_sql/model/mixins.py

@@ -59,5 +59,7 @@ def __new__(
         if not args or (len(args) >= 2 and isinstance(args[1], sa.MetaData)):
             return super().__new__(cls, *args, **kwargs)
 
-        metadata = make_metadata(__database__ or DEFAULT_KEY)
-        return super().__new__(cls, *[args[0], metadata, *args[1:]], **kwargs)
+        db_metadata = make_metadata(__database__ or DEFAULT_KEY)
+        return super().__new__(
+            cls, *[args[0], db_metadata.metadata, *args[1:]], **kwargs
+        )

ellar_sql/model/table.py

@@ -5,25 +5,32 @@
 
 from ellar_sql.constant import DATABASE_BIND_KEY, DEFAULT_KEY, NAMING_CONVERSION
 
-from .database_binds import get_metadata, has_metadata, update_database_metadata
+from .database_binds import (
+    DatabaseMetadata,
+    get_metadata,
+    has_metadata,
+    update_database_metadata,
+)
 
 
-def make_metadata(database_key: str) -> sa.MetaData:
+def make_metadata(database_key: str) -> DatabaseMetadata:
     if has_metadata(database_key):
         return get_metadata(database_key, certain=True)
 
     if database_key != DEFAULT_KEY:
         # Copy the naming convention from the default metadata.
-        naming_convention = make_metadata(DEFAULT_KEY).naming_convention
+        naming_convention = make_metadata(DEFAULT_KEY).metadata.naming_convention
     else:
         naming_convention = NAMING_CONVERSION
 
     # Set the bind key in info to be used by session.get_bind.
     metadata = sa.MetaData(
         naming_convention=naming_convention, info={DATABASE_BIND_KEY: database_key}
     )
-    update_database_metadata(database_key, metadata)
-    return metadata
+    update_database_metadata(
+        database_key, metadata, registry=sa_orm.registry(metadata=metadata)
+    )
+    return get_metadata(database_key, certain=True)
 
 
 def camel_to_snake_case(name: str) -> str: