Full text search (#193)

Author: JWittmeyer

business_objects/record.py

@@ -981,3 +981,28 @@ def get_records_by_running_ids(project_id: str, running_ids: List[int]) -> List[
         )
         .all()
     )
+
+
+def get_record_data_by_sanitized_where(
+    refinery_project_id: str,
+    sanitized_where: str,
+    limit: int,
+    order_by: Optional[str] = None,
+) -> List[Dict[str, Any]]:
+    ## only to be used in cognition and with sql_validator check!!
+    refinery_project_id = prevent_sql_injection(
+        refinery_project_id, isinstance(refinery_project_id, str)
+    )
+    final_order = ""
+    if order_by:
+        order_by = prevent_sql_injection(order_by, isinstance(order_by, str))
+        final_order = f" ORDER BY {order_by} "
+    query = f"""
+    SELECT r.data::JSON
+    FROM public.record r
+    WHERE project_id = '{refinery_project_id}' AND ({sanitized_where})
+    {final_order}
+    LIMIT {limit}
+    """
+    data = general.execute_all(query)
+    return [row[0] for row in data] if data else []

enums.py

@@ -567,6 +567,7 @@ class StrategyStepType(Enum):
     GRAPHRAG_SEARCH = "GRAPHRAG_SEARCH"
     TEMPLATED = "TEMPLATED"
     RERANKER = "RERANKER"
+    FULL_TEXT_SEARCH = "FULL_TEXT_SEARCH"
 
     def get_description(self):
         return STEP_DESCRIPTIONS.get(self, "No description available")
@@ -596,6 +597,7 @@ def get_progress_text(self):
     StrategyStepType.GRAPHRAG_SEARCH: "Query GraphRAG index",
     StrategyStepType.TEMPLATED: "Templated step",
     StrategyStepType.RERANKER: "Reranker",
+    StrategyStepType.FULL_TEXT_SEARCH: "Full text search",
 }
 
 STEP_WHEN_TO_USE = {
@@ -615,6 +617,7 @@ def get_progress_text(self):
     StrategyStepType.GRAPHRAG_SEARCH: "When you want to query a knowledge graph",
     StrategyStepType.TEMPLATED: "When you want to reuse existing templates",
     StrategyStepType.RERANKER: "When you want to rerank results",
+    StrategyStepType.FULL_TEXT_SEARCH: "When you want to perform a full text search",
 }
 
 STEP_PROGRESS_TEXTS = {
@@ -635,6 +638,7 @@ def get_progress_text(self):
     StrategyStepType.GRAPHRAG_SEARCH: "Querying knowledge graph",
     StrategyStepType.TEMPLATED: "Running templated step",
     StrategyStepType.RERANKER: "Running reranker",
+    StrategyStepType.FULL_TEXT_SEARCH: "Running full text search",
 }
 
 STEP_ERRORS = {