add models db

Author: CrysR

src/webapp/database.py

@@ -114,9 +114,7 @@ class InstTable(Base):
     account_histories: Mapped[List["AccountHistoryTable"]] = relationship(
         back_populates="inst"
     )
-    # models: Mapped[List["ModelTable"]] = relationship(
-    #    back_populates="inst"
-    # )
+    models: Mapped[Set["ModelTable"]] = relationship(back_populates="inst")
 
     name = Column(String(VAR_CHAR_LONGER_LENGTH), nullable=False, unique=True)
     # If retention unset, the Datakind default is used. File-level retentions overrides
@@ -208,7 +206,7 @@ class AccountTable(Base):
     # Required for team integration with laravel
     current_team_id = Column(Uuid(as_uuid=True), nullable=True)
     access_type = Column(String(VAR_CHAR_LENGTH), nullable=True)
-    profile_photo_path = Column(String(VAR_CHAR_LENGTH), nullable=True)
+    # profile_photo_path = Column(String(VAR_CHAR_LENGTH), nullable=True)
     created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
     updated_at = Column(DateTime(timezone=True), onupdate=func.now())
 
@@ -314,7 +312,7 @@ class BatchTable(Base):
     name = Column(String(VAR_CHAR_LONGER_LENGTH), nullable=False)
     # A short description or note on this inst.
     description = Column(String(VAR_CHAR_LONGER_LENGTH))
-    creator = Column(Uuid(as_uuid=True))
+    created_by = Column(Uuid(as_uuid=True))
     # If null, the following is non-deleted.
     deleted: Mapped[bool] = mapped_column(nullable=True)
     # If true, the batch is ready for use.
@@ -327,7 +325,6 @@ class BatchTable(Base):
     __table_args__ = (UniqueConstraint("name", "inst_id", name="batch_name_inst_uc"),)
 
 
-"""
 class ModelTable(Base):
     __tablename__ = "model"
     id = Column(Uuid(as_uuid=True), primary_key=True, default=uuid.uuid4)
@@ -341,43 +338,31 @@ class ModelTable(Base):
     inst: Mapped["InstTable"] = relationship(back_populates="models")
 
     name = Column(String(VAR_CHAR_LONGER_LENGTH), nullable=False)
-    # A short description or note on this inst.
-    description = Column(String(VAR_CHAR_LONGER_LENGTH))
-    # What configuration of schemas are allowed (e.g. 1 PDP Course + 1 PDP Cohort)
-    schema_configs = Column(MutableList.as_mutable(JSON))
-    creator = Column(Uuid(as_uuid=True))
+    # A short description or note on this model.
+    description = Column(String(VAR_CHAR_LONGER_LENGTH), nullable=True)
+    # What configuration of schemas are allowed (list of maps e.g. [PDP Course : 1 + PDP Cohort : 1, X_schema :1 + Y_schema: 2])
+    schema_configs = Column(MutableList.as_mutable(JSON), nullable=True)
+    # A list of all the runs executed using this model. These ids will correspond to Databricks ids so that we can retrieve things like
+    # status and correlate output using Databricks.
+    run_ids = Column(MutableList.as_mutable(JSON), nullable=True)
+    created_by = Column(Uuid(as_uuid=True), nullable=True)
     # If null, the following is non-deleted.
     deleted: Mapped[bool] = mapped_column(nullable=True)
-    # If true, the model is ready for use.
-    active: Mapped[bool] = mapped_column(nullable=True)
+    # If true, the model has been approved and is ready for use.
+    valid: Mapped[bool] = mapped_column(nullable=True)
     # The time the deletion request was set.
     deleted_at = Column(DateTime(timezone=True), nullable=True)
     created_at = Column(DateTime(timezone=True), server_default=func.now())
     updated_at = Column(DateTime(timezone=True), onupdate=func.now())
-    # Within a given institution, there should be no duplicated model names.
-    __table_args__ = (UniqueConstraint("name", "inst_id", name="model_name_inst_uc"),)
+    # Each new version of a model can have the same name, but note the uuid will be different.
+    version = Column(Integer, default=0)
 
-class ModelRunsTable(Base):
-    __tablename__ = "model_runs"
-    id = Column(Uuid(as_uuid=True), primary_key=True, default=uuid.uuid4)
-
-    # Set the parent foreign key to link to the institution table.
-    inst_id = Column(
-        Uuid(as_uuid=True),
-        ForeignKey("inst.id", ondelete="CASCADE"),
-        nullable=False,
+    # Within a given institution, there should be no duplicated model names + versions.
+    __table_args__ = (
+        UniqueConstraint("name", "inst_id", "version", name="model_name_inst_uc"),
     )
-    inst: Mapped["InstTable"] = relationship(back_populates="batches")
 
-    # Set the parent foreign key to link to the institution table.
-    inst_id = Column(
-        Uuid(as_uuid=True),
-        ForeignKey("inst.id", ondelete="CASCADE"),
-        nullable=False,
-    )
-    inst: Mapped["InstTable"] = relationship(back_populates="batches")
 
-"""
 """
 def get_one_record(sess_context_var: ContextVar, sess: Session, select_query: ) -> Any:
     local_session.set(sql_session)
@@ -394,7 +379,7 @@ def get_one_record(sess_context_var: ContextVar, sess: Session, select_query: )
                 name=req.name,
                 inst_id=inst_id,
                 description=req.description,
-                creator=current_user.user_id,
+                created_by=current_user.user_id,
             )
         )
         local_session.get().commit()

src/webapp/routers/data.py

@@ -62,7 +62,7 @@ class BatchInfo(BaseModel):
     name: str | None = None
     description: str | None = None
     # User id of uploader or person who triggered this data ingestion.
-    creator: str | None = None
+    created_by: str | None = None
     # Deleted data means this batch has a pending deletion request and can no longer be used.
     deleted: bool | None = None
     # Completed batches means this batch is ready for use. Completed batches will
@@ -215,7 +215,7 @@ def get_all_batches(
                 "name": elem.name,
                 "description": elem.description,
                 "file_names_to_ids": {x.name: uuid_to_str(x.id) for x in elem.files},
-                "creator": uuid_to_str(elem.creator),
+                "created_by": uuid_to_str(elem.created_by),
                 "deleted": False if elem.deleted is None else elem.deleted,
                 "completed": False if elem.completed is None else elem.completed,
                 "deletion_request_time": elem.deleted_at,
@@ -351,7 +351,7 @@ def read_batch_info(
         "name": res.name,
         "description": res.description,
         "file_names_to_ids": {x.name: uuid_to_str(x.id) for x in res.files},
-        "creator": uuid_to_str(res.creator),
+        "created_by": uuid_to_str(res.created_by),
         "deleted": False if res.deleted is None else res.deleted,
         "completed": False if res.completed is None else res.completed,
         "deletion_request_time": res.deleted_at,
@@ -415,13 +415,13 @@ def create_batch(
             name=req.name,
             inst_id=str_to_uuid(inst_id),
             description=req.description,
-            creator=str_to_uuid(current_user.user_id),
+            created_by=str_to_uuid(current_user.user_id),
         )
         f_names = [] if not req.file_names else req.file_names
         f_ids = [] if not req.file_ids else strs_to_uuids(req.file_ids)
         # Check that the files requested for this batch exists.
         # Only valid non-sst generated files can be added to a batch at creation time.
-        query_result_file = (
+        query_result_files = (
             local_session.get()
             .execute(
                 select(FileTable).where(
@@ -438,17 +438,13 @@ def create_batch(
             )
             .all()
         )
-        if not query_result_file or len(query_result_file) == 0:
+        if not query_result_files or len(query_result_files) == 0:
             raise HTTPException(
                 status_code=status.HTTP_404_NOT_FOUND,
                 detail="file in request not found.",
             )
-        elif len(query_result_file) > 1:
-            raise HTTPException(
-                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                detail="Multiple files in request with same unique id found.",
-            )
-        batch.files.add(query_result_file[0][0])
+        for elem in query_result_files:
+            batch.files.add(elem[0])
         local_session.get().add(batch)
         local_session.get().commit()
         query_result = (
@@ -486,7 +482,7 @@ def create_batch(
         "file_names_to_ids": {
             x.name: uuid_to_str(x.id) for x in query_result[0][0].files
         },
-        "creator": uuid_to_str(query_result[0][0].creator),
+        "created_by": uuid_to_str(query_result[0][0].created_by),
         "deleted": False,
         "completed": False,
         "deletion_request_time": None,
@@ -641,7 +637,7 @@ def update_batch(
         "name": res[0][0].name,
         "description": res[0][0].description,
         "file_names_to_ids": {x.name: uuid_to_str(x.id) for x in res[0][0].files},
-        "creator": uuid_to_str(res[0][0].creator),
+        "created_by": uuid_to_str(res[0][0].created_by),
         "deleted": res[0][0].deleted,
         "completed": res[0][0].completed,
         "deletion_request_time": res[0][0].deleted_at,

src/webapp/routers/data_test.py

@@ -17,6 +17,7 @@
     USER_UUID,
     UUID_INVALID,
     DATETIME_TESTING,
+    SAMPLE_UUID,
 )
 from ..main import app
 from ..database import (
@@ -80,7 +81,7 @@ def same_orderless(a: DataOverview, b: DataOverview):
                 or a_elem["file_names_to_ids"] == b_elem["file_names_to_ids"]
                 or a_elem["name"] != b_elem["name"]
                 or a_elem["description"] != b_elem["description"]
-                or a_elem["creator"] != b_elem["creator"]
+                or a_elem["created_by"] != b_elem["created_by"]
                 or a_elem["deleted"] != b_elem["deleted"]
                 or a_elem["completed"] != b_elem["completed"]
                 or a_elem["deletion_request_time"] != b_elem["deletion_request_time"]
@@ -116,7 +117,7 @@ def session_fixture():
         id=BATCH_UUID,
         inst_id=USER_VALID_INST_UUID,
         name="batch_foo",
-        creator=CREATOR_UUID,
+        created_by=CREATOR_UUID,
         created_at=DATETIME_TESTING,
         updated_at=DATETIME_TESTING,
     )
@@ -135,14 +136,24 @@ def session_fixture():
     file_3 = FileTable(
         id=FILE_UUID_3,
         inst_id=USER_VALID_INST_UUID,
-        name="file_output_one",
+        name="file_output_three",
         batches={batch_1},
         created_at=DATETIME_TESTING,
         updated_at=DATETIME_TESTING,
         sst_generated=True,
         valid=True,
         schemas=[SchemaType.PDP_COHORT],
     )
+    file_4 = FileTable(
+        id=SAMPLE_UUID,
+        inst_id=USER_VALID_INST_UUID,
+        name="file_output_four",
+        created_at=DATETIME_TESTING,
+        updated_at=DATETIME_TESTING,
+        sst_generated=True,
+        valid=True,
+        schemas=[SchemaType.PDP_COHORT],
+    )
     try:
         with sqlalchemy.orm.Session(engine) as session:
             session.add_all(
@@ -167,6 +178,7 @@ def session_fixture():
                         schemas=[SchemaType.PDP_COURSE],
                     ),
                     file_3,
+                    file_4,
                 ]
             )
             session.commit()
@@ -223,7 +235,7 @@ def test_read_inst_all_input_files(client: TestClient):
                     ],
                     "name": "batch_foo",
                     "description": None,
-                    "creator": "0ad8b77c49fb459a84b18d2c05722c4a",
+                    "created_by": "0ad8b77c49fb459a84b18d2c05722c4a",
                     "deleted": False,
                     "completed": False,
                     "deletion_request_time": None,
@@ -289,11 +301,11 @@ def test_read_inst_all_output_files(client: TestClient):
                     "inst_id": "1d7c75c33eda42949c6675ea8af97b55",
                     "file_names_to_ids": [
                         {"file_input_one": "f0bb3a206d924254afed6a72f43c562a"},
-                        {"file_output_one": "fbe67a2e50e040c7b7b807043cb813a5"},
+                        {"file_output_three": "fbe67a2e50e040c7b7b807043cb813a5"},
                     ],
                     "name": "batch_foo",
                     "description": None,
-                    "creator": "0ad8b77c49fb459a84b18d2c05722c4a",
+                    "created_by": "0ad8b77c49fb459a84b18d2c05722c4a",
                     "deleted": False,
                     "completed": False,
                     "deletion_request_time": None,
@@ -302,7 +314,7 @@ def test_read_inst_all_output_files(client: TestClient):
             ],
             "files": [
                 {
-                    "name": "file_output_one",
+                    "name": "file_output_three",
                     "data_id": "fbe67a2e50e040c7b7b807043cb813a5",
                     "batch_ids": ["5b2420f3103546ab90eb74d5df97de43"],
                     "inst_id": "1d7c75c33eda42949c6675ea8af97b55",
@@ -315,7 +327,22 @@ def test_read_inst_all_output_files(client: TestClient):
                     "sst_generated": True,
                     "valid": True,
                     "uploaded_date": "2024-12-24T20:22:20.132022",
-                }
+                },
+                {
+                    "name": "file_output_four",
+                    "data_id": "e4862c62829440d8ab4c9c298f02f619",
+                    "batch_ids": [],
+                    "inst_id": "1d7c75c33eda42949c6675ea8af97b55",
+                    "description": None,
+                    "uploader": "",
+                    "source": None,
+                    "deleted": False,
+                    "deletion_request_time": None,
+                    "retention_days": None,
+                    "sst_generated": True,
+                    "valid": True,
+                    "uploaded_date": "2024-12-24T20:22:20.132022",
+                },
             ],
         },
     )
@@ -352,11 +379,11 @@ def test_read_batch_info(client: TestClient):
                     "inst_id": "1d7c75c33eda42949c6675ea8af97b55",
                     "file_names_to_ids": [
                         {"file_input_one": "f0bb3a206d924254afed6a72f43c562a"},
-                        {"file_output_one": "fbe67a2e50e040c7b7b807043cb813a5"},
+                        {"file_output_three": "fbe67a2e50e040c7b7b807043cb813a5"},
                     ],
                     "name": "batch_foo",
                     "description": None,
-                    "creator": "0ad8b77c49fb459a84b18d2c05722c4a",
+                    "created_by": "0ad8b77c49fb459a84b18d2c05722c4a",
                     "deleted": False,
                     "completed": False,
                     "deletion_request_time": None,
@@ -365,7 +392,7 @@ def test_read_batch_info(client: TestClient):
             ],
             "files": [
                 {
-                    "name": "file_output_one",
+                    "name": "file_output_three",
                     "data_id": "fbe67a2e50e040c7b7b807043cb813a5",
                     "batch_ids": ["5b2420f3103546ab90eb74d5df97de43"],
                     "inst_id": "1d7c75c33eda42949c6675ea8af97b55",
@@ -461,13 +488,13 @@ def test_create_batch(client: TestClient):
             "description": "",
             "batch_disabled": "False",
             "file_ids": [uuid_to_str(FILE_UUID_1)],
-            "file_names": ["file_input_one", "file_input_two"],
+            "file_names": ["file_input_one", "file_input_two", "file_input_four"],
         },
     )
     assert response.status_code == 200
     assert response.json()["name"] == "batch_foobar"
     assert response.json()["description"] == ""
-    assert response.json()["creator"] == uuid_to_str(USER_UUID)
+    assert response.json()["created_by"] == uuid_to_str(USER_UUID)
     assert response.json()["deleted"] == False
     assert response.json()["completed"] == False
     assert response.json()["deletion_request_time"] == None
@@ -511,7 +538,7 @@ def test_update_batch(client: TestClient):
     assert response.status_code == 200
     assert response.json()["name"] == "batch_name_updated_foo"
     assert response.json()["description"] == None
-    assert response.json()["creator"] == uuid_to_str(CREATOR_UUID)
+    assert response.json()["created_by"] == uuid_to_str(CREATOR_UUID)
     assert response.json()["deleted"] == None
     assert response.json()["completed"] == True
     assert response.json()["deletion_request_time"] == None