release/v0.57.0

api/oss/src/models/api/evaluation_model.py

@@ -292,7 +292,6 @@ class LMProvidersEnum(str, Enum):
 
 
 class NewEvaluation(BaseModel):
-    app_id: str
     name: Optional[str] = None
     revisions_ids: List[str]
     evaluators_configs: List[str]
@@ -302,7 +301,6 @@ class NewEvaluation(BaseModel):
 
 
 class NewEvaluatorConfig(BaseModel):
-    app_id: str
     name: str
     evaluator_key: str
     settings_values: dict

api/oss/src/routers/evaluators_router.py

@@ -1,4 +1,4 @@
-from typing import List
+from typing import List, Optional
 from fastapi import HTTPException, Request
 from fastapi.responses import JSONResponse
 
@@ -102,8 +102,8 @@ async def evaluator_run(
 
 @router.get("/configs/", response_model=List[EvaluatorConfig])
 async def get_evaluator_configs(
-    app_id: str,
     request: Request,
+    app_id: Optional[str] = None,
 ):
     """Endpoint to fetch evaluator configurations for a specific app.
 
@@ -113,25 +113,33 @@ async def get_evaluator_configs(
     Returns:
         List[EvaluatorConfigDB]: A list of evaluator configuration objects.
     """
+    project_id: Optional[str] = None
+    if app_id:
+        app_db = await db_manager.fetch_app_by_id(app_id=app_id)
+        project_id = str(app_db.project_id)
+    else:
+        project_id = getattr(request.state, "project_id", None)
+        if project_id is None:
+            raise HTTPException(status_code=400, detail="project_id is required")
 
-    app_db = await db_manager.fetch_app_by_id(app_id=app_id)
     if is_ee():
         has_permission = await check_action_access(
             user_uid=request.state.user_id,
-            project_id=str(app_db.project_id),
+            project_id=project_id,
             permission=Permission.VIEW_EVALUATION,
         )
         if not has_permission:
-            error_msg = f"You do not have permission to perform this action. Please contact your organization admin."
+            error_msg = (
+                "You do not have permission to perform this action. "
+                "Please contact your organization admin."
+            )
             log.error(error_msg)
             return JSONResponse(
                 {"detail": error_msg},
                 status_code=403,
             )
 
-    evaluators_configs = await evaluator_manager.get_evaluators_configs(
-        str(app_db.project_id)
-    )
+    evaluators_configs = await evaluator_manager.get_evaluators_configs(project_id)
     return evaluators_configs
 
 
@@ -181,14 +189,10 @@ async def create_new_evaluator_config(
         EvaluatorConfigDB: Evaluator configuration api model.
     """
 
-    app_db = await db_manager.get_app_instance_by_id(
-        project_id=request.state.project_id,
-        app_id=payload.app_id,
-    )
     if is_ee():
         has_permission = await check_action_access(
             user_uid=request.state.user_id,
-            project_id=str(app_db.project_id),
+            project_id=request.state.project_id,
             permission=Permission.CREATE_EVALUATION,
         )
         if not has_permission:
@@ -200,8 +204,7 @@ async def create_new_evaluator_config(
             )
 
     evaluator_config = await evaluator_manager.create_evaluator_config(
-        project_id=str(app_db.project_id),
-        app_name=app_db.app_name,
+        project_id=request.state.project_id,
         name=payload.name,
         evaluator_key=payload.evaluator_key,
         settings_values=payload.settings_values,

api/oss/src/services/db_manager.py

@@ -3034,16 +3034,14 @@ async def create_evaluator_config(
     project_id: str,
     name: str,
     evaluator_key: str,
-    app_name: Optional[str] = None,
     settings_values: Optional[Dict[str, Any]] = None,
 ) -> EvaluatorConfigDB:
     """Create a new evaluator configuration in the database."""
 
     async with engine.core_session() as session:
-        name_suffix = f" ({app_name})" if app_name else ""
         new_evaluator_config = EvaluatorConfigDB(
             project_id=uuid.UUID(project_id),
-            name=f"{name}{name_suffix}",
+            name=name,
             evaluator_key=evaluator_key,
             settings_values=settings_values,
         )

api/oss/src/services/evaluator_manager.py

@@ -54,7 +54,6 @@ async def get_evaluator_config(evaluator_config: EvaluatorConfig) -> EvaluatorCo
 
 async def create_evaluator_config(
     project_id: str,
-    app_name: str,
     name: str,
     evaluator_key: str,
     settings_values: Optional[Dict[str, Any]] = None,
@@ -75,7 +74,6 @@ async def create_evaluator_config(
 
     evaluator_config = await db_manager.create_evaluator_config(
         project_id=project_id,
-        app_name=app_name,
         name=name,
         evaluator_key=evaluator_key,
         settings_values=settings_values,
@@ -149,7 +147,6 @@ async def create_ready_to_use_evaluators(project_id: str):
         ), f"'name' and 'key' does not exist in the evaluator: {evaluator}"
         await db_manager.create_evaluator_config(
             project_id=project_id,
-            app_name=None,
             name=evaluator.name,
             evaluator_key=evaluator.key,
             settings_values=settings_values,

api/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "api"
-version = "0.56.4"
+version = "0.57.0"
 description = "Agenta API"
 authors = [
     { name = "Mahmoud Mabrouk", email = "[email protected]" },

docs/docs/reference/openapi.json

@@ -12706,10 +12706,6 @@
       },
       "NewEvaluation": {
         "properties": {
-          "app_id": {
-            "type": "string",
-            "title": "App Id"
-          },
           "revisions_ids": {
             "items": {
               "type": "string"
@@ -12744,15 +12740,11 @@
           }
         },
         "type": "object",
-        "required": ["app_id", "revisions_ids", "evaluators_configs", "testset_id", "rate_limit"],
+        "required": ["revisions_ids", "evaluators_configs", "testset_id", "rate_limit"],
         "title": "NewEvaluation"
       },
       "NewEvaluatorConfig": {
         "properties": {
-          "app_id": {
-            "type": "string",
-            "title": "App Id"
-          },
           "name": {
             "type": "string",
             "title": "Name"
@@ -12768,7 +12760,7 @@
           }
         },
         "type": "object",
-        "required": ["app_id", "name", "evaluator_key", "settings_values"],
+        "required": ["name", "evaluator_key", "settings_values"],
         "title": "NewEvaluatorConfig"
       },
       "NewHumanEvaluation": {

sdk/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "agenta"
-version = "0.56.4"
+version = "0.57.0"
 description = "The SDK for agenta is an open-source LLMOps platform."
 readme = "README.md"
 authors = [

web/oss/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@agenta/oss",
-    "version": "0.56.4",
+    "version": "0.57.0",
     "private": true,
     "engines": {
         "node": ">=18"

web/oss/src/components/ErrorState/index.tsx

@@ -0,0 +1,33 @@
+import {Result, Button} from "antd"
+import type {FC} from "react"
+
+interface ErrorStateProps {
+    title?: string
+    subtitle?: string
+    status?: "error" | "warning" | "info" | "500"
+    onRetry?: () => void
+}
+
+const ErrorState: FC<ErrorStateProps> = ({
+    title = "Something went wrong",
+    subtitle = "Please try again",
+    status = "error",
+    onRetry,
+}) => {
+    return (
+        <Result
+            status={status}
+            title={title}
+            subTitle={subtitle}
+            extra={
+                onRetry ? (
+                    <Button type="primary" onClick={onRetry} data-testid="error-retry">
+                        Retry
+                    </Button>
+                ) : null
+            }
+        />
+    )
+}
+
+export default ErrorState

web/oss/src/components/Layout/Layout.tsx

@@ -131,6 +131,7 @@ const AppWithVariants = memo(
                                     className={clsx(classes.content, {
                                         "[&.ant-layout-content]:p-0 [&.ant-layout-content]:m-0":
                                             isPlayground,
+                                        "flex flex-col min-h-0 grow": isHumanEval,
                                     })}
                                 >
                                     <ErrorBoundary FallbackComponent={ErrorFallback}>

web/oss/src/components/Playground/Components/MainLayout/index.tsx

@@ -1,5 +1,5 @@
 import React from "react"
-import {memo, useCallback, useRef} from "react"
+import {memo, useCallback, useEffect, useRef} from "react"
 
 import {Typography, Button, Splitter} from "antd"
 import clsx from "clsx"
@@ -75,6 +75,26 @@ const PlaygroundMainView = ({className, isLoading = false, ...divProps}: MainLay
         enabled: isComparisonView,
     })
 
+    useEffect(() => {
+        if (process.env.NODE_ENV !== "production") {
+            console.info("[PlaygroundMainView] state", {
+                displayedVariants,
+                isComparisonView,
+                shouldShowVariantConfigSkeleton,
+                shouldShowGenerationSkeleton,
+                appStatus,
+                appStatusLoading,
+            })
+        }
+    }, [
+        displayedVariants,
+        isComparisonView,
+        shouldShowVariantConfigSkeleton,
+        shouldShowGenerationSkeleton,
+        appStatus,
+        appStatusLoading,
+    ])
+
     const handleScroll = useCallback(
         (index: number) => {
             const targetRef = variantRefs.current[index]

web/oss/src/components/Playground/Components/PlaygroundVariantConfig/assets/Editors.tsx

@@ -1,5 +1,6 @@
-import {memo} from "react"
+import {useEffect, memo} from "react"
 
+import {Spin} from "antd"
 import clsx from "clsx"
 
 import {useVariantPrompts} from "@/oss/components/Playground/hooks/useVariantPrompts"
@@ -15,7 +16,27 @@ const PlaygroundVariantConfigEditors = ({
     variantId: string
     className?: string
 }) => {
-    const {promptIds} = useVariantPrompts(variantId)
+    const {promptIds, variantExists, debug} = useVariantPrompts(variantId)
+
+    useEffect(() => {
+        if (process.env.NODE_ENV !== "production") {
+            console.info("[PlaygroundVariantConfigEditors]", {
+                variantId,
+                promptCount: promptIds.length,
+                variantExists,
+                debug,
+            })
+        }
+    }, [variantId, promptIds.length, variantExists, debug])
+
+    if (!variantExists) {
+        return (
+            <div className="flex flex-col items-center justify-center py-6 gap-2">
+                <Spin tip="Loading variant configuration…" />
+                <span className="text-xs text-gray-500">variantId: {variantId}</span>
+            </div>
+        )
+    }
 
     return (
         <div className={clsx("flex flex-col", className)} {...divProps}>

web/oss/src/components/Playground/Components/PlaygroundVariantModelConfig/index.tsx

@@ -81,21 +81,24 @@ const PlaygroundVariantModelConfig: React.FC<PlaygroundVariantModelConfigProps>
         }
     }, [variantId, promptId, propertyIds])
 
-    const displayModel = resolvedModelName ?? rawModelName ?? "Choose a model"
+    const displayModel = resolvedModelName ?? "Choose a model"
     const canOpen = (propertyIds?.length || 0) > 0
 
-    const handleOpenChange = useCallback(
-        (open: boolean) => {
-            setIsModalOpen(open)
-        },
-        [variantId, promptId, viewOnly, propertyIds, modelPropertyId, resolvedModelName],
-    )
-
     return (
         <Popover
             {...popoverProps} // Pass through Popover props
-            open={canOpen ? isModalOpen : false}
-            onOpenChange={canOpen ? handleOpenChange : undefined}
+            open={canOpen && !viewOnly ? isModalOpen : false}
+            onOpenChange={
+                canOpen
+                    ? (open) => {
+                          if (viewOnly) {
+                              setIsModalOpen(false)
+                              return
+                          }
+                          setIsModalOpen(open)
+                      }
+                    : undefined
+            }
             classNames={{
                 root: "w-full max-w-[300px]",
             }}
@@ -119,7 +122,7 @@ const PlaygroundVariantModelConfig: React.FC<PlaygroundVariantModelConfigProps>
             }
             className={className}
         >
-            <Button onClick={handleModalOpen} disabled={!canOpen}>
+            <Button onClick={handleModalOpen} disabled={!canOpen || viewOnly}>
                 {displayModel} <CaretDown size={14} />
             </Button>
         </Popover>