Remove gates (#253)

* Remove gates

* remove refinery pat

* New alembic

* New alembic

* shema change

* fix: schema update in downgrade step

* Removal of crowd labeler and zeroshot (#254)

* Removal of crowd labeler and zeroshot

* Remove zero-shot unused code

* Removed model callbacks

* Removed model callbacks on data browser and filters

* Model callback removal

* Users filter removed on data browser

* Removed label studio

* Remove doc_ock from refinery

* Removed dummy alembic test code

* Submodule change

---------

Co-authored-by: andhreljaKern <[email protected]>
Co-authored-by: lumburovskalina <[email protected]>

Author: 3 people

alembic/versions/414c990688f3_remove_refinery_token_table.py

@@ -0,0 +1,103 @@
+"""Remove refinery token table
+
+Revision ID: 414c990688f3
+Revises: 3e59ce51739c
+Create Date: 2024-09-09 09:25:36.796509
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '414c990688f3'
+down_revision = '3e59ce51739c'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    # generated code
+    op.drop_index('ix_personal_access_token_project_id', table_name='personal_access_token')
+    op.drop_index('ix_personal_access_token_user_id', table_name='personal_access_token')
+    op.drop_table('personal_access_token')
+
+    op.drop_index('ix_cognition_project_refinery_question_project_id', table_name='project', schema='cognition')
+    op.drop_index('ix_cognition_project_refinery_references_project_id', table_name='project', schema='cognition')
+    op.drop_index('ix_cognition_project_refinery_relevance_project_id', table_name='project', schema='cognition')
+    op.drop_constraint('project_refinery_references_project_id_fkey', 'project', schema='cognition', type_='foreignkey')
+    op.drop_constraint('project_refinery_question_project_id_fkey', 'project', schema='cognition', type_='foreignkey')
+    op.drop_constraint('project_refinery_relevance_project_id_fkey', 'project', schema='cognition', type_='foreignkey')
+    op.drop_column('project', 'refinery_references_project_id', schema='cognition')
+    op.drop_column('project', 'refinery_synchronization_interval_option', schema='cognition')
+    op.drop_column('project', 'refinery_question_project_id', schema='cognition')
+    op.drop_column('project', 'refinery_relevance_project_id', schema='cognition')
+    op.drop_column('project', 'execute_query_enrichment_if_source_code', schema='cognition')
+    
+    op.drop_index('ix_cognition_refinery_synchronization_task_cognition_project_id', table_name='refinery_synchronization_task', schema='cognition')
+    op.drop_index('ix_cognition_refinery_synchronization_task_created_by', table_name='refinery_synchronization_task', schema='cognition')
+    op.drop_index('ix_cognition_refinery_synchronization_task_refinery_project_id', table_name='refinery_synchronization_task', schema='cognition')
+    op.drop_table('refinery_synchronization_task', schema='cognition')
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    
+    # ------------------------ pat remove ------------------------
+    op.create_table('personal_access_token',
+    sa.Column('id', postgresql.UUID(), autoincrement=False, nullable=False),
+    sa.Column('project_id', postgresql.UUID(), autoincrement=False, nullable=True),
+    sa.Column('user_id', postgresql.UUID(), autoincrement=False, nullable=True),
+    sa.Column('name', sa.VARCHAR(), autoincrement=False, nullable=True),
+    sa.Column('scope', sa.VARCHAR(), autoincrement=False, nullable=True),
+    sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True),
+    sa.Column('expires_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True),
+    sa.Column('last_used', postgresql.TIMESTAMP(), autoincrement=False, nullable=True),
+    sa.Column('token', sa.VARCHAR(), autoincrement=False, nullable=True),
+    sa.ForeignKeyConstraint(['project_id'], ['project.id'], name='personal_access_token_project_id_fkey', ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['user_id'], ['user.id'], name='personal_access_token_user_id_fkey', ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id', name='personal_access_token_pkey')
+    )
+    op.create_index('ix_personal_access_token_user_id', 'personal_access_token', ['user_id'], unique=False)
+    op.create_index('ix_personal_access_token_project_id', 'personal_access_token', ['project_id'], unique=False)
+
+    # ------------------------ cognition table fields ------------------------
+
+    op.add_column('project', sa.Column('execute_query_enrichment_if_source_code', sa.VARCHAR(), autoincrement=False, nullable=True), schema='cognition')
+    op.add_column('project', sa.Column('refinery_relevance_project_id', postgresql.UUID(), autoincrement=False, nullable=True), schema='cognition')
+    op.add_column('project', sa.Column('refinery_question_project_id', postgresql.UUID(), autoincrement=False, nullable=True), schema='cognition')
+    op.add_column('project', sa.Column('refinery_synchronization_interval_option', sa.VARCHAR(), autoincrement=False, nullable=True), schema='cognition')
+    op.add_column('project', sa.Column('refinery_references_project_id', postgresql.UUID(), autoincrement=False, nullable=True), schema='cognition')
+    op.create_foreign_key('project_refinery_relevance_project_id_fkey', 'project', 'project', ['refinery_relevance_project_id'], ['id'], source_schema='cognition', ondelete='SET NULL')
+    op.create_foreign_key('project_refinery_question_project_id_fkey', 'project', 'project', ['refinery_question_project_id'], ['id'], source_schema='cognition', ondelete='SET NULL')
+    op.create_foreign_key('project_refinery_references_project_id_fkey', 'project', 'project', ['refinery_references_project_id'], ['id'], source_schema='cognition', ondelete='SET NULL')
+    op.create_index('ix_cognition_project_refinery_relevance_project_id', 'project', ['refinery_relevance_project_id'], unique=False, schema='cognition')
+    op.create_index('ix_cognition_project_refinery_references_project_id', 'project', ['refinery_references_project_id'], unique=False, schema='cognition')
+    op.create_index('ix_cognition_project_refinery_question_project_id', 'project', ['refinery_question_project_id'], unique=False, schema='cognition')
+    
+    # ------------------------ sync table ------------------------	
+
+    op.create_table('refinery_synchronization_task',
+    sa.Column('id', postgresql.UUID(), autoincrement=False, nullable=False),
+    sa.Column('cognition_project_id', postgresql.UUID(), autoincrement=False, nullable=True),
+    sa.Column('refinery_project_id', postgresql.UUID(), autoincrement=False, nullable=True),
+    sa.Column('created_by', postgresql.UUID(), autoincrement=False, nullable=True),
+    sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True),
+    sa.Column('finished_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True),
+    sa.Column('state', sa.VARCHAR(), autoincrement=False, nullable=True),
+    sa.Column('logs', postgresql.ARRAY(sa.VARCHAR()), autoincrement=False, nullable=True),
+    sa.Column('num_records_created', sa.INTEGER(), autoincrement=False, nullable=True),
+    sa.ForeignKeyConstraint(['cognition_project_id'], ['cognition.project.id'], name='refinery_synchronization_task_cognition_project_id_fkey', ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['created_by'], ['user.id'], name='refinery_synchronization_task_created_by_fkey', ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['refinery_project_id'], ['project.id'], name='refinery_synchronization_task_refinery_project_id_fkey', ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id', name='refinery_synchronization_task_pkey'),
+    schema='cognition'
+    )
+    op.create_index('ix_cognition_refinery_synchronization_task_refinery_project_id', 'refinery_synchronization_task', ['refinery_project_id'], unique=False, schema='cognition')
+    op.create_index('ix_cognition_refinery_synchronization_task_created_by', 'refinery_synchronization_task', ['created_by'], unique=False, schema='cognition')
+    op.create_index('ix_cognition_refinery_synchronization_task_cognition_project_id', 'refinery_synchronization_task', ['cognition_project_id'], unique=False, schema='cognition')
+
+    # ### end Alembic commands ###

api/project.py

@@ -7,12 +7,6 @@
 from controller.attribute import manager as attribute_manager
 from submodules.model import exceptions
 
-from submodules.model import events
-from util import doc_ock, notification, adapter
-
-from controller.task_master import manager as task_master_manager
-from submodules.model.enums import TaskType, RecordTokenizationScope
-
 logging.basicConfig(level=logging.DEBUG)
 
 
@@ -48,54 +42,3 @@ def get(self, request) -> JSONResponse:
             "knowledge_base_ids": [str(list.id) for list in project.knowledge_bases],
         }
         return JSONResponse(result)
-
-
-class ProjectCreationFromWorkflow(HTTPEndpoint):
-    async def post(self, request_body) -> JSONResponse:
-        (
-            user_id,
-            name,
-            description,
-            tokenizer,
-            store_id,
-        ) = await adapter.unpack_request_body(request_body)
-
-        user = auth_manager.get_user_by_id(user_id)
-        organization = auth_manager.get_organization_by_user_id(user.id)
-
-        project = project_manager.create_project(
-            str(organization.id), name, description, user.id
-        )
-        project_manager.update_project(project_id=project.id, tokenizer=tokenizer)
-        data = adapter.get_records_from_store(store_id)
-        adapter.check(data, project.id, user.id)
-
-        project_manager.add_workflow_store_data_to_project(
-            user_id=user.id,
-            project_id=project.id,
-            org_id=project.organization_id,
-            file_name=name,
-            data=data,
-        )
-
-        task_master_manager.queue_task(
-            str(organization.id),
-            str(user.id),
-            TaskType.TOKENIZATION,
-            {
-                "scope": RecordTokenizationScope.PROJECT.value,
-                "include_rats": True,
-                "only_uploaded_attributes": False,
-                "project_id": str(project.id),
-            },
-        )
-
-        notification.send_organization_update(
-            project.id, f"project_created:{str(project.id)}", True
-        )
-        doc_ock.post_event(
-            str(user.id),
-            events.CreateProject(Name=f"{name}-{project.id}", Description=description),
-        )
-
-        return JSONResponse({"project_id": str(project.id)})

api/transfer.py

@@ -6,7 +6,6 @@
 from starlette.responses import PlainTextResponse, JSONResponse
 from controller.embedding.manager import recreate_embeddings
 
-from controller.transfer.labelstudio import import_preperator
 from controller.transfer.cognition import (
     import_preparator as cognition_preparator,
     import_wizard as cognition_import_wizard,
@@ -244,6 +243,9 @@ def put(self, request) -> PlainTextResponse:
 
 class AssociationsImport(HTTPEndpoint):
     async def post(self, request) -> JSONResponse:
+        # Will be removed as part of the python sdk removal
+        return JSONResponse({"error": "Not supported anymore"}, status_code=404)
+
         project_id = request.path_params["project_id"]
         request_body = await request.json()
         user_id = request_body["user_id"]
@@ -298,9 +300,6 @@ def init_file_import(task: UploadTask, project_id: str, is_global_update: bool)
     if "records" in task.file_type:
         if task.upload_type == enums.UploadTypes.COGNITION.value:
             cognition_preparator.prepare_cognition_import(project_id, task)
-        elif task.upload_type == enums.UploadTypes.LABEL_STUDIO.value:
-            # deprecated
-            import_preperator.prepare_label_studio_import(project_id, task)
         else:
             transfer_manager.import_records_from_file(project_id, task)
         daemon.run(

app.py

@@ -4,7 +4,7 @@
 from api.healthcheck import Healthcheck
 from starlette.middleware import Middleware
 from api.misc import IsDemoRest, IsManagedRest
-from api.project import ProjectDetails, ProjectCreationFromWorkflow
+from api.project import ProjectDetails
 from api.transfer import (
     AssociationsImport,
     FileExport,
@@ -21,7 +21,6 @@
 from fast_api.routes.project_setting import router as project_setting_router
 from fast_api.routes.misc import router as misc_router
 from fast_api.routes.comment import router as comment_router
-from fast_api.routes.zero_shot import router as zero_shot_router
 from fast_api.routes.attribute import router as attribute_router
 from fast_api.routes.embedding import router as embedding_router
 from fast_api.routes.notification import router as notification_router
@@ -47,7 +46,6 @@
     PREFIX_PROJECT_SETTING,
     PREFIX_MISC,
     PREFIX_COMMENT,
-    PREFIX_ZERO_SHOT,
     PREFIX_ATTRIBUTE,
     PREFIX_EMBEDDING,
     PREFIX_NOTIFICATION,
@@ -79,9 +77,6 @@
 )
 fastapi_app.include_router(misc_router, prefix=PREFIX_MISC, tags=["misc"])
 fastapi_app.include_router(comment_router, prefix=PREFIX_COMMENT, tags=["comment"])
-fastapi_app.include_router(
-    zero_shot_router, prefix=PREFIX_ZERO_SHOT, tags=["zero-shot"]
-)
 fastapi_app.include_router(
     attribute_router, prefix=PREFIX_ATTRIBUTE, tags=["attribute"]
 )
@@ -139,7 +134,6 @@
         CognitionPrepareProject,
     ),
     Route("/project/{project_id:str}/import/task/{task_id:str}", UploadTaskInfo),
-    Route("/project", ProjectCreationFromWorkflow),
     Route("/is_managed", IsManagedRest),
     Route("/is_demo", IsDemoRest),
     Mount("/api", app=fastapi_app, name="REST API"),

controller/embedding/manager.py

@@ -50,27 +50,26 @@ def get_recommended_encoders(is_managed: bool) -> List[Any]:
     else:
         existing_models = []
     for model in existing_models:
-        if not model["zero_shot_pipeline"]:
-            not_yet_known = (
-                len(
-                    list(
-                        filter(
-                            lambda rec: rec["config_string"] == model["name"],
-                            recommendations,
-                        )
+        not_yet_known = (
+            len(
+                list(
+                    filter(
+                        lambda rec: rec["config_string"] == model["name"],
+                        recommendations,
                     )
                 )
-                == 0
             )
-            if not_yet_known:
-                recommendations.append(
-                    {
-                        "config_string": model["name"],
-                        "description": "User downloaded model",
-                        "tokenizers": ["all"],
-                        "applicability": {"attribute": True, "token": True},
-                    }
-                )
+            == 0
+        )
+        if not_yet_known:
+            recommendations.append(
+                {
+                    "config_string": model["name"],
+                    "description": "User downloaded model",
+                    "tokenizers": ["all"],
+                    "applicability": {"attribute": True, "token": True},
+                }
+            )
     return recommendations