Merge pull request #774 from parea-ai/PAI-1048-public-api-get-trace-logs-of-experiment

feat: fetch logs for experiment

Author: joschkabraun

parea/client.py

@@ -14,7 +14,7 @@
 from parea.cache.cache import Cache
 from parea.constants import PAREA_OS_ENV_EXPERIMENT_UUID
 from parea.experiment.datasets import create_test_cases, create_test_collection
-from parea.helpers import gen_trace_id, serialize_metadata_values, structure_trace_log_from_api
+from parea.helpers import gen_trace_id, serialize_metadata_values, structure_trace_log_from_api, structure_trace_logs_from_api
 from parea.parea_logger import parea_logger
 from parea.schemas.models import (
     Completion,
@@ -31,6 +31,7 @@
     ProjectSchema,
     TestCaseCollection,
     TraceLog,
+    TraceLogFilters,
     UseDeployedPrompt,
     UseDeployedPromptResponse,
 )
@@ -52,6 +53,7 @@
 ADD_TEST_CASES_ENDPOINT = "/testcases"
 GET_TRACE_LOG_ENDPOINT = "/trace_log/{trace_id}"
 LIST_EXPERIMENTS_ENDPOINT = "/experiments"
+GET_EXPERIMENT_LOGS_ENDPOINT = "/experiment/{experiment_uuid}/trace_logs"
 
 
 @define
@@ -356,6 +358,14 @@ async def alist_experiment_uuids(self, filter_conditions: Optional[ListExperimen
         response = await self._client.request_async("POST", LIST_EXPERIMENTS_ENDPOINT, data=asdict(filter_conditions))
         return response.json()
 
+    def get_experiment_trace_logs(self, experiment_uuid: str, filters: TraceLogFilters = TraceLogFilters()) -> List[TraceLog]:
+        response = self._client.request("POST", GET_EXPERIMENT_LOGS_ENDPOINT.format(experiment_uuid=experiment_uuid), data=asdict(filters))
+        return structure_trace_logs_from_api(response.json())
+
+    async def aget_experiment_trace_logs(self, experiment_uuid: str, filters: TraceLogFilters = TraceLogFilters()) -> List[TraceLog]:
+        response = await self._client.request_async("POST", GET_EXPERIMENT_LOGS_ENDPOINT.format(experiment_uuid=experiment_uuid), data=asdict(filters))
+        return structure_trace_logs_from_api(response.json())
+
 
 _initialized_parea_wrapper = False
 

parea/cookbook/list_experiments.py

@@ -9,6 +9,7 @@
 
 p = Parea(api_key=os.getenv("PAREA_API_KEY"))
 
-exp_uuids = p.list_experiment_uuids(ListExperimentUUIDsFilters())
-print(len(exp_uuids))
-print(exp_uuids)
+exp_uuids = p.list_experiment_uuids(ListExperimentUUIDsFilters(experiment_name_filter="Greeting"))
+print(f"Num. experiments: {len(exp_uuids)}")
+trace_logs = p.get_experiment_trace_logs(exp_uuids[0])
+print(f"Num. trace logs: {len(trace_logs)}")

parea/helpers.py

@@ -93,3 +93,7 @@ def structure_union_type(obj: Any, cl: type) -> Any:
     converter = GenConverter()
     converter.register_structure_hook(Union[str, Dict[str, str], None], structure_union_type)
     return converter.structure(d, TraceLog)
+
+
+def structure_trace_logs_from_api(data: List[dict]) -> List[TraceLog]:
+    return [structure_trace_log_from_api(d) for d in data]

parea/schemas/models.py

@@ -88,6 +88,23 @@ class TraceLogImage:
     caption: Optional[str] = None
 
 
+@define
+class TraceLogCommentSchema:
+    comment: str
+    user_id: str
+    created_at: str
+
+
+@define
+class TraceLogAnnotationSchema:
+    created_at: str
+    user_id: str
+    score: float
+    user_email_address: Optional[str] = None
+    annotation_name: Optional[str] = None
+    value: Optional[str] = None
+
+
 @define
 class TraceLog(EvaluatedLog):
     trace_id: Optional[str] = field(default=None, validator=validators.instance_of(str))
@@ -120,6 +137,10 @@ class TraceLog(EvaluatedLog):
     experiment_uuid: Optional[str] = None
     images: Optional[List[TraceLogImage]] = field(factory=list)
 
+    # from UI
+    comments: Optional[List[TraceLogCommentSchema]] = None
+    annotations: Optional[dict[int, dict[str, TraceLogAnnotationSchema]]] = None
+
 
 @define
 class TraceLogTree(TraceLog):
@@ -286,3 +307,23 @@ class ListExperimentUUIDsFilters:
     metadata_filter: Optional[Dict[str, Any]] = None
     experiment_name_filter: Optional[str] = None
     run_name_filter: Optional[str] = None
+
+
+class FilterOperator(str, Enum):
+    EQUALS = "equals"
+    NOT_EQUALS = "not_equals"
+    LIKE = "like"
+    GREATER_THAN_OR_EQUAL = "greater_than_or_equal"
+    LESS_THAN_OR_EQUAL = "less_than_or_equal"
+    GRATER_THAN = "greater_than"
+    LESS_THAN = "less_than"
+    IS_NULL = "is_null"
+    EXISTS = "exists"
+    IN = "in"
+
+
+@define
+class TraceLogFilters:
+    filter_field: Optional[str] = None
+    filter_operator: Optional[FilterOperator] = None
+    filter_value: Optional[str] = None

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "poetry.core.masonry.api"
 [tool.poetry]
 name = "parea-ai"
 packages = [{ include = "parea" }]
-version = "0.2.133"
+version = "0.2.134"
 description = "Parea python sdk"
 readme = "README.md"
 authors = ["joel-parea-ai <[email protected]>"]