From d2e637f59ed5382b4212d9cf5f75c45dca4a55fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Sat, 11 Nov 2023 14:37:10 +0100 Subject: [PATCH 01/10] etl pipeline --- ..._adds_markdown_file_llm_transformation_.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 alembic/versions/9ec726915d3c_adds_markdown_file_llm_transformation_.py diff --git a/alembic/versions/9ec726915d3c_adds_markdown_file_llm_transformation_.py b/alembic/versions/9ec726915d3c_adds_markdown_file_llm_transformation_.py new file mode 100644 index 00000000..ab91a39e --- /dev/null +++ b/alembic/versions/9ec726915d3c_adds_markdown_file_llm_transformation_.py @@ -0,0 +1,45 @@ +"""adds markdown file llm transformation logs + +Revision ID: 9ec726915d3c +Revises: 491ea68a7baf +Create Date: 2023-11-03 21:20:45.305683 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '9ec726915d3c' +down_revision = '491ea68a7baf' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('markdown_llm_logs', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('markdown_file_id', postgresql.UUID(as_uuid=True), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('finished_at', sa.DateTime(), nullable=True), + sa.Column('input', sa.String(), nullable=True), + sa.Column('output', sa.String(), nullable=True), + sa.Column('error', sa.String(), nullable=True), + sa.ForeignKeyConstraint(['markdown_file_id'], ['cognition.markdown_file.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + schema='cognition' + ) + op.create_index(op.f('ix_cognition_markdown_llm_logs_markdown_file_id'), 'markdown_llm_logs', ['markdown_file_id'], unique=False, schema='cognition') + op.add_column('markdown_file', sa.Column('state', sa.String(), nullable=True), schema='cognition') + op.add_column('markdown_file', sa.Column('finished_at', sa.DateTime(), nullable=True), schema='cognition') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('markdown_file', 'finished_at', schema='cognition') + op.drop_column('markdown_file', 'state', schema='cognition') + op.drop_index(op.f('ix_cognition_markdown_llm_logs_markdown_file_id'), table_name='markdown_llm_logs', schema='cognition') + op.drop_table('markdown_llm_logs', schema='cognition') + # ### end Alembic commands ### From 00a8f5978ab0db6dae5c01c70d2ddfb18d2e7d3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Sun, 12 Nov 2023 19:48:47 +0100 Subject: [PATCH 02/10] etl-pipeline --- .../7498717d42ef_adds_markdown_dataset.py | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 alembic/versions/7498717d42ef_adds_markdown_dataset.py diff --git a/alembic/versions/7498717d42ef_adds_markdown_dataset.py b/alembic/versions/7498717d42ef_adds_markdown_dataset.py new file mode 100644 index 00000000..60742ca5 --- /dev/null +++ b/alembic/versions/7498717d42ef_adds_markdown_dataset.py @@ -0,0 +1,100 @@ +"""adds markdown dataset + +Revision ID: 7498717d42ef +Revises: 9ec726915d3c +Create Date: 2023-11-12 12:07:55.382820 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '7498717d42ef' +down_revision = '9ec726915d3c' +branch_labels = None +depends_on = None + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("markdown_file", sa.Column("dataset_id", postgresql.UUID(as_uuid=True), nullable=True), schema="cognition") + + op.create_table( + "markdown_dataset", + sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("organization_id", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("created_by", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("created_at",sa.DateTime(), nullable=True), + sa.Column("name", sa.String(), nullable=True), + sa.Column("description", sa.String(), nullable=True), + sa.Column("category_origin", sa.String(), nullable=True), + sa.ForeignKeyConstraint( + ["organization_id"], ["organization.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + schema="cognition", + ) + + op.create_foreign_key( + None, + "markdown_file", + "markdown_dataset", + ["dataset_id"], + ["id"], + ondelete="CASCADE", + source_schema="cognition", + referent_schema="cognition" + ) + + op.create_index( + op.f("ix_cognition_markdown_file_dataset_id"), + "markdown_file", + ["dataset_id"], + unique=False, + schema="cognition", + ) + + op.create_index( + op.f("ix_cognition_markdown_dataset_created_by"), + "markdown_dataset", + ["created_by"], + unique=False, + schema="cognition", + ) + + op.create_index( + op.f("ix_cognition_markdown_dataset_organization_id"), + "markdown_dataset", + ["organization_id"], + unique=False, + schema="cognition", + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + + op.drop_index( + op.f("ix_cognition_markdown_file_dataset_id"), + table_name="markdown_file", + schema="cognition", + ) + + op.drop_constraint(None, "markdown_file", type_="foreignkey", schema="cognition") + op.drop_column("markdown_file", "dataset_id", schema="cognition") + + op.drop_index( + op.f("ix_cognition_markdown_dataset_created_by"), + table_name="markdown_dataset", + schema="cognition", + ) + + op.drop_index( + op.f("ix_cognition_markdown_dataset_organization_id"), + table_name="markdown_dataset", + schema="cognition", + ) + op.drop_table("markdown_dataset", schema="cognition") + + # ### end Alembic commands ### From 95c48e53096ae2578555f4f6a102c7fabeec2539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Sun, 12 Nov 2023 21:52:34 +0100 Subject: [PATCH 03/10] tokenizer --- alembic/versions/7498717d42ef_adds_markdown_dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/alembic/versions/7498717d42ef_adds_markdown_dataset.py b/alembic/versions/7498717d42ef_adds_markdown_dataset.py index 60742ca5..a188fe7b 100644 --- a/alembic/versions/7498717d42ef_adds_markdown_dataset.py +++ b/alembic/versions/7498717d42ef_adds_markdown_dataset.py @@ -26,6 +26,7 @@ def upgrade(): sa.Column("created_by", postgresql.UUID(as_uuid=True), nullable=True), sa.Column("created_at",sa.DateTime(), nullable=True), sa.Column("name", sa.String(), nullable=True), + sa.Column("tokenizer", sa.String(), nullable=True), sa.Column("description", sa.String(), nullable=True), sa.Column("category_origin", sa.String(), nullable=True), sa.ForeignKeyConstraint( @@ -83,7 +84,7 @@ def downgrade(): op.drop_constraint(None, "markdown_file", type_="foreignkey", schema="cognition") op.drop_column("markdown_file", "dataset_id", schema="cognition") - + op.drop_index( op.f("ix_cognition_markdown_dataset_created_by"), table_name="markdown_dataset", From d2ba3103f7bf1623879d16197145e431db2d117e Mon Sep 17 00:00:00 2001 From: JWittmeyer Date: Tue, 14 Nov 2023 17:38:55 +0100 Subject: [PATCH 04/10] Removes faulty wizard done state & adds title for generated questions --- controller/transfer/cognition/import_wizard.py | 12 +----------- submodules/model | 2 +- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/controller/transfer/cognition/import_wizard.py b/controller/transfer/cognition/import_wizard.py index 2fa66ee4..919f7535 100644 --- a/controller/transfer/cognition/import_wizard.py +++ b/controller/transfer/cognition/import_wizard.py @@ -148,17 +148,6 @@ def finalize_setup(cognition_project_id: str, task_id: str) -> None: organization_id=organization_id, ) - task_list.append( - { - "project_id": reference_project_id, - "task_type": enums.TaskType.TASK_QUEUE_ACTION.value, - "action": { - "action_type": enums.TaskQueueAction.FINISH_COGNITION_SETUP.value, - "cognition_project_id": cognition_project_id, - }, - } - ) - __add_start_gates_for(reference_project_id, task_list) __add_start_gates_for(question_project_id, task_list) @@ -697,6 +686,7 @@ def __add_records_to_question_and_relevance( final_question = question if "?" not in final_question: final_question += "?" + final_question = final_question[0].title() + final_question[1:] message_id = "mr-" + str(idx) final_json_to_add_questions.append( { diff --git a/submodules/model b/submodules/model index 0d18f4d1..559d6168 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 0d18f4d1d8c952ce572998f360ce509f2ec3494a +Subproject commit 559d6168fb76465fceb82d806e606e5d59190243 From 6009dc773c12c1337a2587830623679e53710046 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Sat, 18 Nov 2023 11:06:44 +0100 Subject: [PATCH 05/10] merge with origin dev --- submodules/model | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/model b/submodules/model index 559d6168..4aac0427 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 559d6168fb76465fceb82d806e606e5d59190243 +Subproject commit 4aac0427056c3b85a3d20494742e387120973547 From 80c158d9eb58cc6a8a574f593eddcb8e593135b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Sat, 18 Nov 2023 15:01:37 +0100 Subject: [PATCH 06/10] begin queue-based processing --- .../7498717d42ef_adds_markdown_dataset.py | 29 +++++++++++-------- api/transfer.py | 18 ++++++++++++ .../task_queue/handler/markdown_file.py | 24 +++++++++++++++ 3 files changed, 59 insertions(+), 12 deletions(-) create mode 100644 controller/task_queue/handler/markdown_file.py diff --git a/alembic/versions/7498717d42ef_adds_markdown_dataset.py b/alembic/versions/7498717d42ef_adds_markdown_dataset.py index a188fe7b..c4916e42 100644 --- a/alembic/versions/7498717d42ef_adds_markdown_dataset.py +++ b/alembic/versions/7498717d42ef_adds_markdown_dataset.py @@ -23,6 +23,7 @@ def upgrade(): "markdown_dataset", sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False), sa.Column("organization_id", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("refinery_project_id", postgresql.UUID(as_uuid=True), nullable=True), sa.Column("created_by", postgresql.UUID(as_uuid=True), nullable=True), sa.Column("created_at",sa.DateTime(), nullable=True), sa.Column("name", sa.String(), nullable=True), @@ -36,17 +37,6 @@ def upgrade(): schema="cognition", ) - op.create_foreign_key( - None, - "markdown_file", - "markdown_dataset", - ["dataset_id"], - ["id"], - ondelete="CASCADE", - source_schema="cognition", - referent_schema="cognition" - ) - op.create_index( op.f("ix_cognition_markdown_file_dataset_id"), "markdown_file", @@ -70,6 +60,15 @@ def upgrade(): unique=False, schema="cognition", ) + + op.create_index( + op.f("ix_cognition_markdown_dataset_refinery_project_id"), + "markdown_dataset", + ["refinery_project_id"], + unique=False, + schema="cognition", + ) + # ### end Alembic commands ### @@ -82,7 +81,6 @@ def downgrade(): schema="cognition", ) - op.drop_constraint(None, "markdown_file", type_="foreignkey", schema="cognition") op.drop_column("markdown_file", "dataset_id", schema="cognition") op.drop_index( @@ -96,6 +94,13 @@ def downgrade(): table_name="markdown_dataset", schema="cognition", ) + + op.drop_index( + op.f("ix_cognition_markdown_dataset_refinery_project_id"), + table_name="markdown_dataset", + schema="cognition", + ) + op.drop_table("markdown_dataset", schema="cognition") # ### end Alembic commands ### diff --git a/api/transfer.py b/api/transfer.py index b527bce9..f067ab37 100644 --- a/api/transfer.py +++ b/api/transfer.py @@ -231,6 +231,24 @@ def put(self, request) -> PlainTextResponse: ) return PlainTextResponse("OK") + + +class CognitionParseMarkdownFile(HTTPEndpoint): + def put(self, request) -> PlainTextResponse: + # cognition_project_id = request.path_params["cognition_project_id"] + + # cognition_project_item = cognition_project.get(cognition_project_id) + # if not cognition_project_item: + # return PlainTextResponse("Bad project id", status_code=400) + # task_id = request.path_params["task_id"] + + # daemon.run( + # cognition_import_wizard.finalize_setup, + # cognition_project_id=cognition_project_id, + # task_id=task_id, + # ) + + return PlainTextResponse("OK") class AssociationsImport(HTTPEndpoint): diff --git a/controller/task_queue/handler/markdown_file.py b/controller/task_queue/handler/markdown_file.py new file mode 100644 index 00000000..7bdf5894 --- /dev/null +++ b/controller/task_queue/handler/markdown_file.py @@ -0,0 +1,24 @@ +from typing import Any, Dict, Tuple, Callable +from submodules.model.business_objects import ( + task_queue as task_queue_db_bo, + general, +) +from submodules.model.cognition_objects import ( + markdown_file as markdown_file_db_bo, +) + + +def get_task_functions() -> Tuple[Callable, Callable, int]: + return __start_task, __check_finished, 1 + + +def __start_task(task: Dict[str, Any]) -> bool: + # check task still relevant + task_db_obj = task_queue_db_bo.get(task["id"]) + if task_db_obj is None or task_db_obj.is_active: + return False + return True + + +def __check_finished(task: Dict[str, Any]) -> bool: + project_id = task["project_id"] From 53a74292f2c5d5c6fdb8fb6063c8b9d99429245c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Sun, 19 Nov 2023 01:09:29 +0100 Subject: [PATCH 07/10] markdown file parsing as etl process --- api/transfer.py | 36 +++++++++++-------- app.py | 5 +++ .../task_queue/handler/markdown_file.py | 22 +++++++++++- controller/task_queue/manager.py | 3 ++ submodules/model | 2 +- 5 files changed, 51 insertions(+), 17 deletions(-) diff --git a/api/transfer.py b/api/transfer.py index f067ab37..754c720e 100644 --- a/api/transfer.py +++ b/api/transfer.py @@ -19,6 +19,7 @@ general, organization, tokenization, + project as refinery_project, ) from submodules.model.cognition_objects import project as cognition_project @@ -38,8 +39,8 @@ from util import daemon, notification from controller.task_queue import manager as task_queue_manager -from submodules.model.enums import TaskType, RecordTokenizationScope - +from submodules.model.enums import TaskType, RecordTokenizationScope, TaskQueueAction +import requests logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) @@ -234,19 +235,24 @@ def put(self, request) -> PlainTextResponse: class CognitionParseMarkdownFile(HTTPEndpoint): - def put(self, request) -> PlainTextResponse: - # cognition_project_id = request.path_params["cognition_project_id"] - - # cognition_project_item = cognition_project.get(cognition_project_id) - # if not cognition_project_item: - # return PlainTextResponse("Bad project id", status_code=400) - # task_id = request.path_params["task_id"] - - # daemon.run( - # cognition_import_wizard.finalize_setup, - # cognition_project_id=cognition_project_id, - # task_id=task_id, - # ) + def post(self, request) -> PlainTextResponse: + refinery_project_id = request.path_params["project_id"] + refinery_project_item = refinery_project.get(refinery_project_id) + if not refinery_project_item: + return PlainTextResponse("Bad project id", status_code=400) + + dataset_id = request.path_params["dataset_id"] + file_id = request.path_params["file_id"] + + task_queue_manager.add_task( + refinery_project_id, + TaskType.PARSE_MARKDOWN_FILE, + refinery_project_item.created_by, + { + "dataset_id": dataset_id, + "file_id": file_id, + }, + ) return PlainTextResponse("OK") diff --git a/app.py b/app.py index 8bc08c78..a1025d02 100644 --- a/app.py +++ b/app.py @@ -13,6 +13,7 @@ UploadTaskInfo, CognitionImport, CognitionPrepareProject, + CognitionParseMarkdownFile, ) from middleware.database_session import DatabaseSessionHandler from starlette.applications import Starlette @@ -55,6 +56,10 @@ "/project/{cognition_project_id:str}/cognition/continue/{task_id:str}/finalize", CognitionPrepareProject, ), + Route( + "/project/{project_id:str}/cognition/datasets/{dataset_id:str}/files/{file_id:str}/queue", + CognitionParseMarkdownFile, + ), Route("/project/{project_id:str}/import/task/{task_id:str}", UploadTaskInfo), Route("/project", ProjectCreationFromWorkflow), Route("/is_managed", IsManagedRest), diff --git a/controller/task_queue/handler/markdown_file.py b/controller/task_queue/handler/markdown_file.py index 7bdf5894..0bb86eaf 100644 --- a/controller/task_queue/handler/markdown_file.py +++ b/controller/task_queue/handler/markdown_file.py @@ -1,4 +1,6 @@ from typing import Any, Dict, Tuple, Callable + +import requests from submodules.model.business_objects import ( task_queue as task_queue_db_bo, general, @@ -6,6 +8,7 @@ from submodules.model.cognition_objects import ( markdown_file as markdown_file_db_bo, ) +from submodules.model import enums def get_task_functions() -> Tuple[Callable, Callable, int]: @@ -17,8 +20,25 @@ def __start_task(task: Dict[str, Any]) -> bool: task_db_obj = task_queue_db_bo.get(task["id"]) if task_db_obj is None or task_db_obj.is_active: return False + + print("Starting markdown file task", flush=True) + action = task["task_info"] + dataset_id = action["dataset_id"] + file_id = action["file_id"] + + task_db_obj.is_active = True + general.commit() + requests.post(f"http://cognition-gateway:80/converters-noop/datasets/{dataset_id}/files/{file_id}/parse") return True def __check_finished(task: Dict[str, Any]) -> bool: - project_id = task["project_id"] + action = task["task_info"] + file_id = action["file_id"] + markdown_file_entity = markdown_file_db_bo.get(file_id) + + if markdown_file_entity.state == enums.CognitionMarkdownFileState.FINISHED.value or markdown_file_entity.state == enums.CognitionMarkdownFileState.FAILED.value: + print("Markdown file finished", flush=True) + return True + else: + return False diff --git a/controller/task_queue/manager.py b/controller/task_queue/manager.py index 91d19f17..c7b4212d 100644 --- a/controller/task_queue/manager.py +++ b/controller/task_queue/manager.py @@ -16,6 +16,7 @@ tokenization as tokenization_handler, attribute_calculation as attribute_calculation_handler, task_queue as task_queue_handler, + markdown_file as markdown_file_handler, ) from .util import if_task_queue_send_websocket @@ -87,6 +88,8 @@ def get_task_function_by_type(task_type: str) -> Tuple[Callable, Callable, int]: return attribute_calculation_handler.get_task_functions() if task_type == enums.TaskType.TASK_QUEUE.value: return task_queue_handler.get_task_functions() + if task_type == enums.TaskType.PARSE_MARKDOWN_FILE.value: + return markdown_file_handler.get_task_functions() raise ValueError(f"Task type {task_type} not supported yet") diff --git a/submodules/model b/submodules/model index 4aac0427..74dac66c 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 4aac0427056c3b85a3d20494742e387120973547 +Subproject commit 74dac66ce9538e05f6aab9fe8ec77bb4337efbf5 From 5742fd4ba89bc01eb645a15447ab217bf656e66a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Sun, 19 Nov 2023 01:10:27 +0100 Subject: [PATCH 08/10] markdown file parsing as etl process --- submodules/model | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/model b/submodules/model index 74dac66c..b2aee19f 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 74dac66ce9538e05f6aab9fe8ec77bb4337efbf5 +Subproject commit b2aee19fbbca3d648244381d68d930aa8158d158 From 5d7db295746c7f500872797071459cab55374915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Sun, 26 Nov 2023 20:16:43 +0100 Subject: [PATCH 09/10] adds started at do markdown file --- .../7498717d42ef_adds_markdown_dataset.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/alembic/versions/7498717d42ef_adds_markdown_dataset.py b/alembic/versions/7498717d42ef_adds_markdown_dataset.py index c4916e42..f8d4a74e 100644 --- a/alembic/versions/7498717d42ef_adds_markdown_dataset.py +++ b/alembic/versions/7498717d42ef_adds_markdown_dataset.py @@ -18,6 +18,7 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.add_column("markdown_file", sa.Column("dataset_id", postgresql.UUID(as_uuid=True), nullable=True), schema="cognition") + op.add_column("markdown_file", sa.Column("started_at", sa.DateTime(), nullable=True), schema="cognition") op.create_table( "markdown_dataset", @@ -37,6 +38,17 @@ def upgrade(): schema="cognition", ) + op.create_foreign_key( + None, + "markdown_file", + "markdown_dataset", + ["dataset_id"], + ["id"], + ondelete="CASCADE", + source_schema="cognition", + referent_schema="cognition" + ) + op.create_index( op.f("ix_cognition_markdown_file_dataset_id"), "markdown_file", @@ -81,7 +93,15 @@ def downgrade(): schema="cognition", ) + op.drop_constraint( + None, + "markdown_file", + type_="foreignkey", + schema="cognition", + ) + op.drop_column("markdown_file", "dataset_id", schema="cognition") + op.drop_column("markdown_file", "started_at", schema="cognition") op.drop_index( op.f("ix_cognition_markdown_dataset_created_by"), From d909a3a7e862587a7f1acb59047d4b1b97997226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Mon, 27 Nov 2023 07:40:33 +0100 Subject: [PATCH 10/10] adds model used --- alembic/versions/7498717d42ef_adds_markdown_dataset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/alembic/versions/7498717d42ef_adds_markdown_dataset.py b/alembic/versions/7498717d42ef_adds_markdown_dataset.py index f8d4a74e..1961a236 100644 --- a/alembic/versions/7498717d42ef_adds_markdown_dataset.py +++ b/alembic/versions/7498717d42ef_adds_markdown_dataset.py @@ -19,6 +19,7 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.add_column("markdown_file", sa.Column("dataset_id", postgresql.UUID(as_uuid=True), nullable=True), schema="cognition") op.add_column("markdown_file", sa.Column("started_at", sa.DateTime(), nullable=True), schema="cognition") + op.add_column("markdown_llm_logs", sa.Column("model_used", sa.String(), nullable=True), schema="cognition") op.create_table( "markdown_dataset", @@ -102,6 +103,7 @@ def downgrade(): op.drop_column("markdown_file", "dataset_id", schema="cognition") op.drop_column("markdown_file", "started_at", schema="cognition") + op.drop_column("markdown_llm_logs", "model_used", schema="cognition") op.drop_index( op.f("ix_cognition_markdown_dataset_created_by"),