Release/0.0.6 (#10)

* Refactor the Plan printing functionality (#8)

* Internal improvements (#9)

* Check all Group attributes are mapped in a Flow

* Use the `.get` function in `to_core` functions

* Rename "Mutation" to "Migration"

* Remove assertions from the `plan` function

* Up package version to 0.0.6

Author: marchinho11

.github/workflows/tests.yaml

@@ -37,7 +37,7 @@ jobs:
           cache: "poetry"
       - run: poetry install
 
-      - run: poetry run pytest --cov=hnhm --cov-report xml --cov-report term tests/
+      - run: poetry run pytest -v --cov=hnhm --cov-report xml --cov-report term tests/
         env:
           PG_DB: ${{ env.PG_DB }}
           PG_USER: ${{ env.PG_USER }}

README.md

@@ -86,7 +86,7 @@ Plan:
     |attribute 'first_name'
     |attribute 'last_name'
 
-Apply mutations? [y/N]: y
+Apply migrations? [y/N]: y
 Applied!
 ```
 

hnhm/cli.py

@@ -3,22 +3,9 @@
 
 import click
 
-from .hnhm import Plan, PlanType
+from .core import HnhmError
+from .plan_printer import print_plan
 from .hnhm_registry import HnhmRegistry
-from .core import (
-    HnhmError,
-    CreateLink,
-    LayoutType,
-    RemoveLink,
-    CreateGroup,
-    RemoveGroup,
-    CreateEntity,
-    RemoveEntity,
-    CreateAttribute,
-    RemoveAttribute,
-    AddGroupAttribute,
-    RemoveGroupAttribute,
-)
 
 
 def import_registry(module: str) -> HnhmRegistry:
@@ -33,91 +20,6 @@ def import_registry(module: str) -> HnhmRegistry:
     return registry
 
 
-def print_plan(plan: Plan):
-    if plan.is_empty():
-        click.secho("Your DWH is up to date.", fg="green")
-        return
-
-    entities_mutations = sorted(plan.entities_mutations.items(), key=lambda kv: kv[0])
-    links_mutations = sorted(plan.links_mutations.items(), key=lambda kv: kv[0])
-
-    click.secho("Plan:")
-    for entity_name, plan_collection in entities_mutations:
-        if plan_collection.type == PlanType.CREATE:
-            symbol, color = "+", "green"
-        elif plan_collection.type == PlanType.REMOVE:
-            symbol, color = "-", "red"
-        elif plan_collection.type == PlanType.UPDATE:
-            symbol, color = "[u]", "yellow"
-        else:
-            raise HnhmError()
-
-        click.secho(f"\n{symbol} entity '{entity_name}'", fg=color)
-        for entity_mutation in plan_collection.mutations:
-            match entity_mutation:
-                case CreateEntity(entity=entity):
-                    if entity.layout.type == LayoutType.HNHM:
-                        click.secho(f"  + hub '{entity.name}'", fg="green")
-                    else:
-                        click.secho(f"  + stage '{entity.name}'", fg="green")
-                        for attribute in entity.attributes.values():
-                            click.secho(f"    |attribute '{attribute.name}'", fg="green")
-
-                case CreateAttribute(entity=_, attribute=attribute):
-                    click.secho(f"  + attribute '{attribute.name}'", fg="green")
-
-                case CreateGroup(entity=_, group=group):
-                    click.secho(f"  + group '{group.name}'", fg="green")
-                    for attribute in group.attributes.values():
-                        click.secho(f"    |attribute '{attribute.name}'", fg="green")
-
-                case AddGroupAttribute(entity=_, group=group, attribute=attribute):
-                    click.secho(f"  [u] group '{group.name}'", fg="yellow")
-                    click.secho(f"    +attribute '{attribute.name}'", fg="green")
-
-                case RemoveEntity(entity=entity):
-                    if entity.layout.type == LayoutType.HNHM:
-                        click.secho(f"  - hub '{entity.name}'", fg="red")
-                    else:
-                        click.secho(f"  - stage '{entity.name}'", fg="red")
-                        for attribute in entity.attributes.values():
-                            click.secho(f"    |attribute '{attribute.name}'", fg="red")
-
-                case RemoveAttribute(entity=_, attribute=attribute):
-                    click.secho(f"  - attribute '{attribute.name}'", fg="red")
-
-                case RemoveGroup(entity=_, group=group):
-                    click.secho(f"  - group '{group.name}'", fg="red")
-                    for attribute in group.attributes.values():
-                        click.secho(f"    | attribute '{attribute.name}'", fg="red")
-
-                case RemoveGroupAttribute(entity=_, group=group, attribute=attribute):
-                    click.secho(f"  [u] group '{group.name}'", fg="yellow")
-                    click.secho(f"    -attribute '{attribute.name}'", fg="red")
-
-    for link_name, plan_collection in links_mutations:
-        if plan_collection.type == PlanType.CREATE:
-            symbol, color = "+", "green"
-        elif plan_collection.type == PlanType.REMOVE:
-            symbol, color = "-", "red"
-        elif plan_collection.type == PlanType.UPDATE:
-            symbol, color = "[u]", "yellow"
-        else:
-            raise HnhmError()
-
-        click.secho(f"\n{symbol} link '{link_name}'", fg=color)
-        for link_mutation in plan_collection.mutations:
-            match link_mutation:
-                case RemoveLink(link=link):
-                    for element in link.elements:
-                        click.secho(f"  |element '{element.entity.name}'", fg=color)
-                case CreateLink(link=link):
-                    for element in link.elements:
-                        click.secho(f"  |element '{element.entity.name}'", fg=color)
-
-    click.secho()
-
-
 @click.group()
 def cli():
     pass
@@ -153,7 +55,7 @@ def apply(dwh_module: str, yes: bool):
     if plan.is_empty():
         return
 
-    if yes or click.confirm("Apply mutations?"):
+    if yes or click.confirm("Apply migrations?"):
         with registry.hnhm as hnhm:
             hnhm.apply(plan)
 

hnhm/core/__init__.py

@@ -7,8 +7,8 @@
 from .attribute import Type, Attribute, ChangeType
 from .storage import Storage, InMemStorage, HnhmStorageData
 from .tasks import Task, LoadHub, LoadLink, LoadGroup, LoadAttribute
-from .mutations import (
-    Mutation,
+from .migrations import (
+    Migration,
     CreateLink,
     RemoveLink,
     CreateGroup,

hnhm/core/mutations.py hnhm/core/migrations.pyhnhm/core/mutations.py renamed to hnhm/core/migrations.py

@@ -7,29 +7,27 @@
 from .attribute import Attribute
 
 
-class Mutation(pydantic.BaseModel):
-    """Mutation represents DWH change."""
-
+class Migration(pydantic.BaseModel):
     priority: Priority
 
 
-class CreateEntity(Mutation):
+class CreateEntity(Migration):
     priority = Priority.FIRST
     entity: Entity
 
     def __str__(self):
         return f"<CreateEntity '{self.entity.name}'>"
 
 
-class RemoveEntity(Mutation):
+class RemoveEntity(Migration):
     priority = Priority.SECOND
     entity: Entity
 
     def __str__(self):
         return f"<RemoveEntity '{self.entity.name}'>"
 
 
-class CreateAttribute(Mutation):
+class CreateAttribute(Migration):
     priority = Priority.SECOND
     entity: Entity
     attribute: Attribute
@@ -38,7 +36,7 @@ def __str__(self):
         return f"<CreateAttribute '{self.attribute.name}' entity='{self.entity.name}'>"
 
 
-class RemoveAttribute(Mutation):
+class RemoveAttribute(Migration):
     priority = Priority.FIRST
     entity: Entity
     attribute: Attribute
@@ -47,7 +45,7 @@ def __str__(self):
         return f"<RemoveAttribute '{self.attribute.name}' entity='{self.entity.name}'>"
 
 
-class CreateGroup(Mutation):
+class CreateGroup(Migration):
     priority = Priority.SECOND
     entity: Entity
     group: Group
@@ -56,7 +54,7 @@ def __str__(self):
         return f"<CreateGroup '{self.group.name}' entity='{self.entity.name}'>"
 
 
-class RemoveGroup(Mutation):
+class RemoveGroup(Migration):
     priority = Priority.FIRST
     entity: Entity
     group: Group
@@ -65,7 +63,7 @@ def __str__(self):
         return f"<RemoveGroup '{self.group.name}' entity='{self.entity.name}'>"
 
 
-class AddGroupAttribute(Mutation):
+class AddGroupAttribute(Migration):
     """Add an Attribute to an existing Group."""
 
     priority = Priority.SECOND
@@ -81,7 +79,7 @@ def __str__(self):
         )
 
 
-class RemoveGroupAttribute(Mutation):
+class RemoveGroupAttribute(Migration):
     """Remove an Attribute from an existing Group."""
 
     priority = Priority.SECOND
@@ -97,15 +95,15 @@ def __str__(self):
         )
 
 
-class CreateLink(Mutation):
+class CreateLink(Migration):
     priority = Priority.SECOND
     link: Link
 
     def __str__(self):
         return f"<CreateLink '{self.link.name}'>"
 
 
-class RemoveLink(Mutation):
+class RemoveLink(Migration):
     priority = Priority.FIRST
     link: Link
 

hnhm/core/sql.py

@@ -1,14 +1,14 @@
 import abc
 
 from .tasks import Task
-from .mutations import Mutation
+from .migrations import Migration
 
 
 class Sql(abc.ABC):
     """Generates and executes sql code."""
 
     @abc.abstractmethod
-    def generate_sql(self, mutation_or_task: Mutation | Task) -> str:
+    def generate_sql(self, migration_or_task: Migration | Task) -> str:
         raise NotImplementedError
 
     @abc.abstractmethod
@@ -17,7 +17,7 @@ def execute(self, sql: str):
 
 
 class FakeSql(Sql):
-    def generate_sql(self, mutation_or_task: Mutation | Task) -> str:
+    def generate_sql(self, migration_or_task: Migration | Task) -> str:
         return ""
 
     def execute(self, sql: str):

hnhm/core/storage.py

@@ -4,6 +4,7 @@
 
 from .link import Link
 from .entity import Entity
+from .error import HnhmError
 
 
 class HnhmStorageData(pydantic.BaseModel):
@@ -12,6 +13,52 @@ class HnhmStorageData(pydantic.BaseModel):
     entities: dict[str, Entity]
     links: dict[str, Link]
 
+    def check_entity_exists(self, entity: str):
+        if entity not in self.entities:
+            raise HnhmError(f"Entity '{entity}' doesn't exist.")
+
+    def check_entity_not_exists(self, entity: str):
+        if entity in self.entities:
+            raise HnhmError(f"Entity '{entity}' already exists.")
+
+    def check_link_exists(self, link: str):
+        if link not in self.links:
+            raise HnhmError(f"Link '{link}' doesn't exist.")
+
+    def check_link_not_exists(self, link: str):
+        if link in self.links:
+            raise HnhmError(f"Link '{link}' already exists.")
+
+    def check_attribute_exists(self, entity: str, attribute: str):
+        self.check_entity_exists(entity)
+        if attribute not in self.entities[entity].attributes:
+            raise HnhmError(f"Attribute '{attribute}' doesn't exist.")
+
+    def check_attribute_not_exists(self, entity: str, attribute: str):
+        self.check_entity_exists(entity)
+        if attribute in self.entities[entity].attributes:
+            raise HnhmError(f"Attribute '{attribute}' already exists.")
+
+    def check_group_exists(self, entity: str, group: str):
+        self.check_entity_exists(entity)
+        if group not in self.entities[entity].groups:
+            raise HnhmError(f"Group '{group}' doesn't exist.")
+
+    def check_group_not_exists(self, entity: str, group: str):
+        self.check_entity_exists(entity)
+        if group in self.entities[entity].groups:
+            raise HnhmError(f"Group '{group}' already exists.")
+
+    def check_group_attribute_exists(self, entity: str, group: str, attribute: str):
+        self.check_group_exists(entity, group)
+        if attribute not in self.entities[entity].groups[group].attributes:
+            raise HnhmError(f"Attribute '{attribute}' doesn't exist.")
+
+    def check_group_attribute_not_exists(self, entity: str, group: str, attribute: str):
+        self.check_group_exists(entity, group)
+        if attribute in self.entities[entity].groups[group].attributes:
+            raise HnhmError(f"Attribute '{attribute}' already exists.")
+
 
 class Storage(abc.ABC):
     @abc.abstractmethod

hnhm/flow.py

@@ -138,10 +138,25 @@ def load(
                         if group_name not in groups_mapping:
                             groups_mapping[group_name] = {}
                         groups_mapping[group_name][attribute_target] = attribute_source
-                        # todo: check all attributes for group were mapped
                     else:
                         attributes_mapping[attribute_target] = attribute_source
 
+                # Check all attributes for a Group were mapped
+                for group_name, group in target_entity.groups.items():
+                    group_mapping = groups_mapping.get(group_name)
+                    if not group_mapping:
+                        continue
+
+                    for attribute in group.attributes.values():
+                        if attribute not in group_mapping:
+                            attribute_full_name = (
+                                f"{target_entity.name}.{group_name}.{attribute.name}"
+                            )
+                            raise HnhmError(
+                                f"Mapping not found for the attribute '{attribute_full_name}'."
+                                f" Please, provide all mappings for the group: '{target_entity.name}.{group_name}'."
+                            )
+
                 missing_keys = set(target_entity.keys) - set(keys_mapping.keys())
                 if missing_keys:
                     missing_keys_names = [key.name for key in missing_keys]