HJ-92: add table for storing monitor execution records (#5704)

Author: thingscouldbeworse

.fides/db_dataset.yml

@@ -916,6 +916,25 @@ dataset:
       data_categories: [system.operations]
     - name: updated_at
       data_categories: [system.operations]
+  - name: monitorexecution
+    data_categories: []
+    fields:
+    - name: id
+      data_categories: [system.operations]
+    - name: monitor_config_key
+      data_categories: [system.operations]
+    - name: status
+      data_categories: [system.operations]
+    - name: started
+      data_categories: [system.operations]
+    - name: completed
+      data_categories: [system.operations]
+    - name: classification_instances
+      data_categories: [system.operations]
+    - name: created_at
+      data_categories: [system.operations]
+    - name: updated_at
+      data_categories: [system.operations]
   - name: policy
     data_categories: []
     fields:

CHANGELOG.md

@@ -47,6 +47,10 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o
 ### Developer Experience
 - Migrated radio buttons and groups to Ant Design [#5681](https://github.com/ethyca/fides/pull/5681)
 
+### Added
+- Migration to add the `data_uses` column to `stagedresource` table, prereqs for Data Catalog work in Fidesplus [#5600](https://github.com/ethyca/fides/pull/5600/) https://github.com/ethyca/fides/labels/db-migration
+- Migration to add the `monitorexecution` table used by fidesplus to persist `MonitorExecution` records to DB [#5704](https://github.com/ethyca/fides/pull/5704) https://github.com/ethyca/fides/labels/db-migration
+
 ### Fixed
 - Updating mongodb connectors so it can support usernames and password with URL encoded characters [#5682](https://github.com/ethyca/fides/pull/5682)
 - After creating a new system, the url is now updated correctly to the new system edit page [#5701](https://github.com/ethyca/fides/pull/5701)

src/fides/api/alembic/migrations/versions/ed96417b07d8_create_table_for_persisting_.py

@@ -0,0 +1,59 @@
+"""create table for persisting MonitorExecution records
+
+Revision ID: ed96417b07d8
+Revises: 1088e8353890
+Create Date: 2025-01-27 23:41:59.803285
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "ed96417b07d8"
+down_revision = "1088e8353890"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "monitorexecution",
+        sa.Column("id", sa.String(), primary_key=True),
+        sa.Column("monitor_config_key", sa.String(), nullable=False),
+        sa.Column("status", sa.String(), nullable=True),
+        sa.Column("started", sa.DateTime(), nullable=True, default=sa.func.now()),
+        sa.Column("completed", sa.DateTime(), nullable=True),
+        sa.Column(
+            "classification_instances",
+            postgresql.ARRAY(sa.String()),
+            nullable=False,
+            default=[],
+        ),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        "ix_monitorexecution_monitor_config_key",
+        "monitorexecution",
+        ["monitor_config_key"],
+    )
+
+
+def downgrade():
+    op.drop_table("monitorexecution")
+    op.drop_index(
+        "ix_monitorexecution_monitor_config_key", table_name="monitorexecution"
+    )

src/fides/api/models/detection_discovery.py

@@ -1,11 +1,11 @@
 from __future__ import annotations
 
-from datetime import datetime
+from datetime import datetime, timezone
 from enum import Enum
 from typing import Any, Dict, Iterable, List, Optional, Type
 
 from loguru import logger
-from sqlalchemy import ARRAY, Boolean, Column, DateTime, ForeignKey, String
+from sqlalchemy import ARRAY, Boolean, Column, DateTime, ForeignKey, String, func
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.ext.mutable import MutableDict
@@ -16,16 +16,6 @@
 from fides.api.db.base_class import Base, FidesBase
 from fides.api.models.connectionconfig import ConnectionConfig
 
-# class MonitorExecution(BaseModel):
-#     id: str
-#     monitor_config_id: str
-#     status: Optional[str]
-#     started: Optional[datetime]
-#     completed: Optional[datetime]
-#     classification_instances: List[str] = PydanticField(
-#         default_factory=list
-#     )  # TODO: formalize to FK
-
 
 class DiffStatus(Enum):
     ADDITION = "addition"
@@ -118,6 +108,12 @@ class MonitorConfig(Base):
 
     connection_config = relationship(ConnectionConfig)
 
+    executions = relationship(
+        "MonitorExecution",
+        cascade="all, delete-orphan",
+        backref="monitor_config",
+    )
+
     @property
     def connection_config_key(self) -> str:
         """Derives the `connection_config_key`"""
@@ -348,6 +344,41 @@ def mark_as_addition(
                 parent_resource.add_child_diff_status(DiffStatus.ADDITION)
 
 
+class MonitorExecution(Base):
+    """
+    Monitor execution record used for data detection and discovery.
+
+    Each monitor execution references `MonitorConfig`, which provide it with underlying
+    configuration details used in connecting to the external data store.
+    """
+
+    id = Column(String, primary_key=True)
+    monitor_config_key = Column(
+        String,
+        ForeignKey(MonitorConfig.key),
+        nullable=False,
+        index=True,
+    )
+    status = Column(String, nullable=True)
+    started = Column(
+        DateTime(timezone=True), nullable=True, default=datetime.now(timezone.utc)
+    )
+    completed = Column(DateTime(timezone=True), nullable=True)
+    classification_instances = Column(
+        ARRAY(String),
+        index=False,
+        unique=False,
+        nullable=False,
+        default=list,
+    )
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(
+        DateTime(timezone=True),
+        server_default=func.now(),
+        onupdate=func.now(),
+    )
+
+
 def fetch_staged_resources_by_type_query(
     resource_type: str,
     monitor_config_ids: Optional[List[str]] = None,