[feature] Add user onboarding

api/oss/src/resources/evaluators/evaluators.py

@@ -203,7 +203,7 @@
     {
         "name": "LLM-as-a-judge",
         "key": "auto_ai_critique",
-        "direct_use": False,
+        "direct_use": True,
         "requires_llm_api_keys": True,
         "settings_presets": [
             {
@@ -221,6 +221,7 @@
                         },
                     ],
                     "model": "gpt-4o-mini",
+                    "template_format": "curly",
                     "response_type": "json_schema",
                     "json_schema": {
                         "name": "schema",
@@ -323,6 +324,13 @@
                 "advanced": True,  # Tells the frontend that this setting is advanced and should be hidden by default
                 "description": "The LLM model to use for the evaluation",
             },
+            "template_format": {
+                "label": "Template Format",
+                "type": "hidden",
+                "default": "curly",
+                "advanced": True,
+                "description": "Formatting style for prompt variables",
+            },
             "response_type": {
                 "label": "Response Type",
                 "default": "json_schema",
@@ -361,6 +369,8 @@
             },
         },
         "description": "LLM-as-a-judge uses a configurable prompt template that takes the output—and optionally inputs or data from the testcase such as correct answer—to evaluate the generated output.",
+        "requires_testcase": "always",
+        "requires_trace": "always",
         "oss": True,
         "tags": ["ai_llm", "functional"],
     },

api/oss/src/services/evaluator_manager.py

@@ -5,6 +5,8 @@
 
 from oss.src.models.db_models import EvaluatorConfigDB
 from oss.src.models.converters import evaluator_config_db_to_pydantic
+import copy
+
 from oss.src.resources.evaluators.evaluators import get_all_evaluators
 from oss.src.models.api.evaluation_model import LegacyEvaluator, EvaluatorConfig
 
@@ -132,15 +134,18 @@ async def create_ready_to_use_evaluators(project_id: str):
 
     for evaluator in direct_use_evaluators:
         settings_values = {
-            setting_name: setting.get("default")
+            setting_name: copy.deepcopy(setting.get("default"))
             for setting_name, setting in evaluator.settings_template.items()
-            if setting.get("ground_truth_key") is True and setting.get("default", "")
+            if "default" in setting
         }
 
         for setting_name, default_value in settings_values.items():
-            assert default_value != "", (
-                f"Default value for ground truth key '{setting_name}' in Evaluator is empty"
-            )
+            if evaluator.settings_template.get(setting_name, {}).get(
+                "ground_truth_key"
+            ):
+                assert default_value != "", (
+                    f"Default value for ground truth key '{setting_name}' in Evaluator is empty"
+                )
 
         assert hasattr(evaluator, "name") and hasattr(evaluator, "key"), (
             f"'name' and 'key' does not exist in the evaluator: {evaluator}"

web/ee/docker/Dockerfile.dev

@@ -18,7 +18,10 @@ COPY ee/package.json ./ee/yarn.lock* ./ee/package-lock.json* ./ee/pnpm-lock.yaml
 COPY oss/package.json ./oss/yarn.lock* ./oss/package-lock.json* ./oss/pnpm-lock.yaml* ./oss/.npmrc* ./oss/
 COPY ./pnpm-workspace.yaml ./turbo.json ./
 
-COPY ./entrypoint.sh /app/entrypoint.sh   
+COPY ./entrypoint.sh /app/entrypoint.sh
+
+# Copy patches directory before running pnpm i
+COPY ./patches /app/patches/
 
 RUN pnpm i
 

web/ee/src/components/DeploymentHistory/DeploymentHistory.tsx

@@ -1,5 +1,6 @@
 import {useCallback, useEffect, useRef, useState} from "react"
 
+import {DeploymentRevisionConfig, DeploymentRevisions} from "@agenta/oss/src/lib/types_ee"
 import {Button, Card, Divider, Space, Typography, notification} from "antd"
 import dayjs from "dayjs"
 import duration from "dayjs/plugin/duration"
@@ -15,8 +16,6 @@ import {
     fetchAllDeploymentRevisions,
 } from "@/oss/services/deploymentVersioning/api"
 
-import {DeploymentRevisionConfig, DeploymentRevisions} from "@agenta/oss/src/lib/types_ee"
-
 dayjs.extend(relativeTime)
 dayjs.extend(duration)
 

web/ee/src/components/PostSignupForm/RunEvaluationView.tsx

@@ -74,7 +74,9 @@ export const RunEvaluationView = () => {
         <div className="flex flex-col gap-8">
             <div className="flex flex-col gap-1">
                 <div className="flex justify-between items-center">
-                    <Title level={1} style={{margin: 0}}>Evaluate from SDK</Title>
+                    <Title level={1} style={{margin: 0}}>
+                        Evaluate from SDK
+                    </Title>
                     <div className="flex items-center gap-2">
                         <Button
                             icon={<Play size={16} className="mt-1" />}
@@ -98,28 +100,26 @@ export const RunEvaluationView = () => {
             </div>
             <ApiKeyInput apiKeyValue={apiKeyValue} onApiKeyChange={setApiKeyValue} />
 
-
             <div className="flex flex-col gap-2">
                 <Text strong>1. Install the required packages:</Text>
-                <TracingCodeComponent 
+                <TracingCodeComponent
                     command={{
                         title: "Bash",
-                        code: "pip install -U agenta"
-                    }} 
-                    index={0} 
+                        code: "pip install -U agenta",
+                    }}
+                    index={0}
                 />
             </div>
 
             <div className="flex flex-col gap-4">
-                <TracingCodeComponent 
+                <TracingCodeComponent
                     command={{
                         title: "Python Code",
-                        code: CODE_SNIPPET
-                    }} 
-                    index={1} 
+                        code: CODE_SNIPPET,
+                    }}
+                    index={1}
                 />
             </div>
-
         </div>
     )
 }

web/ee/src/components/pages/settings/Billing/Modals/PricingModal/assets/SubscriptionPlanDetails/index.tsx

@@ -1,7 +1,7 @@
-import dayjs from "@/oss/lib/helpers/dateTimeHelper/dayjs"
 import duration from "dayjs/plugin/duration"
 import relativeTime from "dayjs/plugin/relativeTime"
 
+import dayjs from "@/oss/lib/helpers/dateTimeHelper/dayjs"
 import {SubscriptionType} from "@/oss/services/billing/types"
 
 dayjs.extend(duration)

web/ee/src/components/pages/settings/Billing/index.tsx

@@ -104,7 +104,7 @@ const Billing = () => {
 
                 <div className="w-full grid grid-cols-3 gap-4">
                     {Object.entries(usage)
-                        ?.filter(([key]) => (key !== "users" && key !== "applications"))
+                        ?.filter(([key]) => key !== "users" && key !== "applications")
                         ?.map(([key, info]) => {
                             return (
                                 <UsageProgressBar

web/oss/docker/Dockerfile.dev

@@ -17,6 +17,8 @@ RUN PNPM_VERSION=$(cat package.json | jq -r '.packageManager | split("@")[1]') &
 
 COPY oss/package.json ./oss/yarn.lock* ./oss/package-lock.json* ./oss/pnpm-lock.yaml* ./oss/.npmrc* ./oss/
 COPY ./pnpm-workspace.yaml ./turbo.json ./
+# Copy patches directory before running pnpm i
+COPY ./patches /app/patches/
 RUN pnpm i
 
 COPY tsconfig.json .

web/oss/package.json

@@ -97,7 +97,9 @@
         "lexical": "^0.38.2",
         "lodash": "^4.17.21",
         "lucide-react": "^0.475.0",
+        "motion": "^12.23.24",
         "next": "15.5.2",
+        "nextstepjs": "^2.1.2",
         "papaparse": "^5.5.3",
         "postcss": "^8.5.6",
         "postcss-antd-fixes": "^0.2.0",

web/oss/src/components/Common/ImagePreview.tsx

@@ -28,35 +28,34 @@ const ImagePreview = ({
     alt = "Preview",
     size = 48,
     className = "",
-    isValidPreview=true,
+    isValidPreview = true,
 }: ImagePreviewProps) => {
-
     const [open, setOpen] = useState(false)
 
     const isSafeImageSrc = (url: string) => {
         if (!url) return false
         // Only allow valid https/http image URLs, blob URLs, or safe data:image URLs
         try {
             // Block javascript: and other schemes
-            const lower = url.toLowerCase().trim();
+            const lower = url.toLowerCase().trim()
             // Only allow https/http with proper image extensions
             if (/^https?:\/\/[^ "]+$/i.test(lower)) {
                 // Optional: Allow only image file extensions
                 if (/\.(png|jpe?g|gif|webp)(\?.*)?$/i.test(lower)) {
                     // Further checking could be done here (e.g., image mimetype fetch), but for now file extension is checked
-                    return true;
+                    return true
                 }
                 // Otherwise, reject
-                return false;
+                return false
             }
             // Allow blob: URLs (browser-generated, controlled)
-            if (lower.startsWith("blob:")) return true;
+            if (lower.startsWith("blob:")) return true
             // Only allow specific data:image/*;base64 URLs
-            if (/^data:image\/(png|jpe?g|gif|webp);base64,[a-z0-9+/=]+$/i.test(lower)) return true;
+            if (/^data:image\/(png|jpe?g|gif|webp);base64,[a-z0-9+/=]+$/i.test(lower)) return true
             // Otherwise, reject
-            return false;
+            return false
         } catch {
-            return false;
+            return false
         }
     }
 
@@ -97,7 +96,11 @@ const ImagePreview = ({
                 height={600}
             >
                 {isValidPreview && isSafeImageSrc(imageURL) && (
-                    <img src={imageURL} alt={alt} className="w-full h-full max-h-[600px] max-w-[800px] object-contain" />
+                    <img
+                        src={imageURL}
+                        alt={alt}
+                        className="w-full h-full max-h-[600px] max-w-[800px] object-contain"
+                    />
                 )}
             </Modal>
         </>

web/oss/src/components/DeploymentsDashboard/components/Drawer/index.tsx

@@ -3,13 +3,14 @@ import {ComponentProps, ReactNode, useState} from "react"
 import {CloseOutlined, FullscreenExitOutlined, FullscreenOutlined} from "@ant-design/icons"
 import {Button, Divider, Drawer} from "antd"
 import clsx from "clsx"
-import {useAtomValue} from "jotai"
+import {useAtomValue, useSetAtom} from "jotai"
 import {createUseStyles} from "react-jss"
 
 import {envRevisionsAtom} from "@/oss/components/DeploymentsDashboard/atoms"
 import EnhancedDrawer from "@/oss/components/EnhancedUIs/Drawer"
 import {usePlaygroundNavigation} from "@/oss/hooks/usePlaygroundNavigation"
 import {JSSTheme} from "@/oss/lib/Types"
+import {openSelectDeployVariantModalAtom} from "@/oss/components/DeploymentsDashboard/modals/store/deploymentModalsStore"
 import {revisionListAtom} from "@/oss/state/variant/selectors/variant"
 
 import UseApiContent from "../../assets/UseApiContent"
@@ -99,6 +100,8 @@ const DeploymentsDrawerContent = ({
 }: DeploymentsDrawerProps) => {
     const variants = useAtomValue(revisionListAtom) || []
     const envRevisions = useAtomValue(envRevisionsAtom)
+    const openSelectDeployVariantModal = useSetAtom(openSelectDeployVariantModalAtom)
+
     return (
         <div className="flex h-full">
             <div className={`flex-1 overflow-auto ${mainContentClassName}`}>
@@ -119,7 +122,9 @@ const DeploymentsDrawerContent = ({
                 >
                     {envRevisions ? (
                         <UseApiContent
-                            handleOpenSelectDeployVariantModal={() => close()}
+                            handleOpenSelectDeployVariantModal={() =>
+                                openSelectDeployVariantModal({variants, envRevisions})
+                            }
                             variants={variants}
                             revisionId={drawerVariantId}
                             selectedEnvironment={envRevisions}

web/oss/src/components/DeploymentsDashboard/index.tsx

@@ -98,6 +98,7 @@ const DeploymentsDashboard: FC<DeploymentsDashboardProps> = ({
                     <Button
                         type="primary"
                         onClick={() => envRevisions && openDeploymentsDrawer({initialWidth: 720})}
+                        id="tour-setup-prompt"
                     >
                         Use API
                     </Button>

web/oss/src/components/Layout/Layout.tsx

@@ -22,11 +22,15 @@ import CustomWorkflowBanner from "../CustomWorkflowBanner"
 import ProtectedRoute from "../ProtectedRoute/ProtectedRoute"
 
 import BreadcrumbContainer from "./assets/Breadcrumbs"
-import {useStyles} from "./assets/styles"
+import {StyleProps, useStyles} from "./assets/styles"
 import ErrorFallback from "./ErrorFallback"
 import {SidebarIsland} from "./SidebarIsland"
 import {getDeviceTheme, useAppTheme} from "./ThemeContextProvider"
 
+const OnboardingWidget = dynamic(() => import("../Onboarding/components/OnboardingWidget"), {
+    ssr: false,
+    loading: () => null,
+})
 const FooterIsland = dynamic(() => import("./FooterIsland").then((m) => m.FooterIsland), {
     ssr: false,
     loading: () => null,
@@ -267,6 +271,7 @@ const App: React.FC<LayoutProps> = ({children}) => {
                         {children}
                         {contextHolder}
                     </AppWithVariants>
+                    <OnboardingWidget />
                 </ProtectedRoute>
             )}
         </Suspense>