release/v0.59.4

.husky/pre-commit

@@ -0,0 +1,15 @@
+#!/usr/bin/env sh
+
+# Delegate to pre-commit which runs configured hooks (e.g. gitleaks) from .pre-commit-config.yaml
+if command -v pre-commit >/dev/null 2>&1; then
+  pre-commit run --hook-stage pre-commit
+else
+  echo "pre-commit not found. Install with one of the following:"
+  echo "  pip install pre-commit   # cross-platform"
+  echo "  brew install pre-commit  # macOS/Linux with Homebrew"
+  echo ""
+  echo "gitleaks is also required. Install with:"
+  echo "  brew install gitleaks    # macOS/Linux with Homebrew"
+  echo "  or see https://github.com/gitleaks/gitleaks#installation for other options"
+  exit 1
+fi

.husky/pre-push

@@ -0,0 +1,12 @@
+#!/usr/bin/env sh
+
+# Delegate to pre-commit which runs pre-push hooks (e.g. gitleaks) from .pre-commit-config.yaml
+if command -v pre-commit >/dev/null 2>&1; then
+  pre-commit run --hook-stage pre-push
+else
+  echo "pre-commit not found. Install with one of the following methods:"
+  echo "  pip install pre-commit gitleaks"
+  echo "  brew install pre-commit gitleaks"
+  echo "  (see https://pre-commit.com/#installation and https://github.com/gitleaks/gitleaks#installation for more options)"
+  exit 1
+fi

api/oss/src/core/services/v0.py

@@ -91,17 +91,27 @@ def _format_with_template(
         elif format == "curly":
             import re
 
+            # Extract variables that exist in the original template before replacement
+            # This allows us to distinguish template variables from {{}} in user input values
+            original_variables = set(re.findall(r"\{\{(.*?)\}\}", content))
+
             result = content
             for key, value in kwargs.items():
                 pattern = r"\{\{" + re.escape(key) + r"\}\}"
                 old_result = result
-                result = re.sub(pattern, str(value), result)
+                # Escape backslashes in the replacement string to prevent regex interpretation
+                escaped_value = str(value).replace("\\", "\\\\")
+                result = re.sub(pattern, escaped_value, result)
+
+            # Only check if ORIGINAL template variables remain unreplaced
+            # Don't error on {{}} that came from user input values
+            unreplaced_matches = set(re.findall(r"\{\{(.*?)\}\}", result))
+            truly_unreplaced = original_variables & unreplaced_matches
 
-            unreplaced_matches = re.findall(r"\{\{(.*?)\}\}", result)
-            if unreplaced_matches:
-                log.info(f"WORKFLOW Found unreplaced variables: {unreplaced_matches}")
+            if truly_unreplaced:
+                log.info(f"WORKFLOW Found unreplaced variables: {truly_unreplaced}")
                 raise ValueError(
-                    f"Template variables not found in inputs: {', '.join(unreplaced_matches)}"
+                    f"Template variables not found in inputs: {', '.join(sorted(truly_unreplaced))}"
                 )
 
             return result

api/oss/src/dbs/postgres/tracing/utils.py

@@ -1935,7 +1935,11 @@ def compute_cqvs(
 
     for k, v in PSC_ITEMS:
         if v[0] in pcts and v[1] in pcts:
-            pscs[k] = (pcts[v[1]] - pcts[v[0]]) / (pcts[v[1]] + pcts[v[0]])
+            pscs[k] = (
+                (pcts[v[1]] - pcts[v[0]]) / (pcts[v[1]] + pcts[v[0]])
+                if (pcts[v[1]] + pcts[v[0]]) != 0
+                else 0.0
+            )
 
     value["pscs"] = pscs
 
@@ -1961,7 +1965,9 @@ def compute_pscs(
 
     for k, v in PSC_ITEMS:
         if v[0] in pcts and v[1] in pcts:
-            pscs[k] = (pcts[v[1]] - pcts[v[0]]) / pcts["p50"]
+            pscs[k] = (
+                (pcts[v[1]] - pcts[v[0]]) / pcts["p50"] if pcts["p50"] != 0 else 0.0
+            )
 
     value["pscs"] = pscs
 

api/oss/src/services/evaluators_service.py

@@ -533,15 +533,26 @@ def _format_with_template(
         elif format == "curly":
             import re
 
+            # Extract variables that exist in the original template before replacement
+            # This allows us to distinguish template variables from {{}} in user input values
+            original_variables = set(re.findall(r"\{\{(.*?)\}\}", content))
+
             result = content
             for key, value in kwargs.items():
                 pattern = r"\{\{" + re.escape(key) + r"\}\}"
                 old_result = result
-                result = re.sub(pattern, str(value), result)
-            unreplaced_matches = re.findall(r"\{\{(.*?)\}\}", result)
-            if unreplaced_matches:
+                # Escape backslashes in the replacement string to prevent regex interpretation
+                escaped_value = str(value).replace("\\", "\\\\")
+                result = re.sub(pattern, escaped_value, result)
+
+            # Only check if ORIGINAL template variables remain unreplaced
+            # Don't error on {{}} that came from user input values
+            unreplaced_matches = set(re.findall(r"\{\{(.*?)\}\}", result))
+            truly_unreplaced = original_variables & unreplaced_matches
+
+            if truly_unreplaced:
                 raise ValueError(
-                    f"Template variables not found in inputs: {', '.join(unreplaced_matches)}"
+                    f"Template variables not found in inputs: {', '.join(sorted(truly_unreplaced))}"
                 )
 
             return result

api/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "api"
-version = "0.59.3"
+version = "0.59.4"
 description = "Agenta API"
 authors = [
     { name = "Mahmoud Mabrouk", email = "[email protected]" },

sdk/agenta/sdk/types.py

@@ -521,13 +521,26 @@ def _format_with_template(self, content: str, kwargs: Dict[str, Any]) -> str:
             elif self.template_format == "curly":
                 import re
 
+                # Extract variables that exist in the original template before replacement
+                # This allows us to distinguish template variables from {{}} in user input values
+                original_variables = set(re.findall(r"\{\{(.*?)\}\}", content))
+
                 result = content
                 for key, value in kwargs.items():
-                    result = re.sub(r"\{\{" + key + r"\}\}", str(value), result)
-                if re.search(r"\{\{.*?\}\}", result):
-                    unreplaced = re.findall(r"\{\{(.*?)\}\}", result)
+                    # Escape backslashes in the replacement string to prevent regex interpretation
+                    escaped_value = str(value).replace("\\", "\\\\")
+                    result = re.sub(
+                        r"\{\{" + re.escape(key) + r"\}\}", escaped_value, result
+                    )
+
+                # Only check if ORIGINAL template variables remain unreplaced
+                # Don't error on {{}} that came from user input values
+                unreplaced_matches = set(re.findall(r"\{\{(.*?)\}\}", result))
+                truly_unreplaced = original_variables & unreplaced_matches
+
+                if truly_unreplaced:
                     raise TemplateFormatError(
-                        f"Unreplaced variables in curly template: {unreplaced}"
+                        f"Unreplaced variables in curly template: {sorted(truly_unreplaced)}"
                     )
                 return result
             else:

sdk/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "agenta"
-version = "0.59.3"
+version = "0.59.4"
 description = "The SDK for agenta is an open-source LLMOps platform."
 readme = "README.md"
 authors = [

web/.husky/install.mjs

@@ -0,0 +1,6 @@
+// Skip Husky install in production and CI
+if (process.env.NODE_ENV === "production" || process.env.CI === "true") {
+    process.exit(0)
+}
+const husky = (await import("husky")).default
+console.log(husky())

web/ee/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@agenta/ee",
-    "version": "0.59.3",
+    "version": "0.59.4",
     "private": true,
     "engines": {
         "node": ">=18"

web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/index.tsx

@@ -4,7 +4,7 @@ import SimpleSharedEditor from "@agenta/oss/src/components/EditorViews/SimpleSha
 import VirtualizedSharedEditors from "@agenta/oss/src/components/EditorViews/VirtualizedSharedEditors"
 import {Collapse, CollapseProps, Tag, Tooltip} from "antd"
 import clsx from "clsx"
-import {useAtomValue} from "jotai"
+import {getDefaultStore, useAtomValue} from "jotai"
 import {loadable} from "jotai/utils"
 import {useRouter} from "next/router"
 
@@ -17,7 +17,6 @@ import {urlStateAtom} from "@/oss/components/EvalRunDetails/state/urlState"
 import {formatMetricValue} from "@/oss/components/HumanEvaluations/assets/MetricDetailsPopover/assets/utils"
 import {getStatusLabel} from "@/oss/lib/constants/statusLabels"
 import {
-    evalAtomStore,
     scenarioStepFamily,
     evaluationRunStateFamily,
 } from "@/oss/lib/hooks/useEvaluationRunData/assets/atoms"
@@ -519,7 +518,7 @@ const FocusDrawerContent = () => {
 
                               // Helper: collect evaluator list for a run
                               const getRunEvaluators = (rId: string) => {
-                                  const rState = evalAtomStore().get(evaluationRunStateFamily(rId))
+                                  const rState = getDefaultStore().get(evaluationRunStateFamily(rId))
                                   const evaluators = rState?.enrichedRun?.evaluators || []
                                   return Array.isArray(evaluators)
                                       ? evaluators
@@ -555,7 +554,7 @@ const FocusDrawerContent = () => {
                                       )
                                   }
 
-                                  const metricData = evalAtomStore().get(
+                                  const metricData = getDefaultStore().get(
                                       runScopedMetricDataFamily({
                                           runId: rId,
                                           scenarioId: scId,
@@ -566,7 +565,7 @@ const FocusDrawerContent = () => {
 
                                   // Run-scoped error fallback
                                   let errorStep: any = null
-                                  const stepLoadableR = evalAtomStore().get(
+                                  const stepLoadableR = getDefaultStore().get(
                                       loadable(scenarioStepFamily({runId: rId, scenarioId: scId})),
                                   ) as any
                                   if (stepLoadableR?.state === "hasData") {
@@ -793,7 +792,8 @@ const FocusDrawerContent = () => {
                               },
                           ),
                           children: Object.keys(metrics || {})?.map((metricKey) => {
-                              const metricData = evalAtomStore().get(
+
+                              const metricData = getDefaultStore().get(
                                   runScopedMetricDataFamily({
                                       runId: runId!,
                                       scenarioId: scenarioId!,
@@ -802,10 +802,10 @@ const FocusDrawerContent = () => {
                                   }),
                               )
 
-                              const errorStep =
-                                  !metricData?.distInfo || hasError
-                                      ? getErrorStep(`${evaluator.slug}.${metricKey}`, scenarioId)
-                                      : null
+                              const errorStep = getErrorStep(
+                                  `${evaluator.slug}.${metricKey}`,
+                                  scenarioId,
+                              )
 
                               let value
                               if (
@@ -825,7 +825,6 @@ const FocusDrawerContent = () => {
                               }
 
                               const formatted = formatMetricValue(metricKey, value || "")
-
                               return (
                                   <div
                                       key={metricKey}
@@ -881,6 +880,7 @@ const FocusDrawerContent = () => {
         matchedComparisonScenarios,
         baseRunId,
         invocationStep?.stepkey,
+        getErrorStep,
     ])
 
     if (stepLoadable.state !== "hasData" || !enricedRun) {

web/ee/src/components/EvalRunDetails/AutoEvalRun/index.tsx

@@ -31,7 +31,7 @@ const AutoEvalRunDetails = ({name, description, id, isLoading}: AutoEvalRunDetai
     return (
         <section
             className={clsx([
-                "flex flex-col w-full h-[calc(100vh-84px)] gap-2 overflow-auto",
+                "flex flex-col w-full !h-[calc(100vh-84px)] gap-2 overflow-auto",
                 {"!overflow-hidden": viewType === "test-cases"},
             ])}
         >

web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ComparisonScenarioTable.tsx

@@ -3,7 +3,7 @@ import {RefObject, useEffect, useMemo} from "react"
 import {DownOutlined} from "@ant-design/icons"
 import clsx from "clsx"
 import {atom, useAtom, useAtomValue} from "jotai"
-import dynamic from "next/dynamic"
+
 import {useResizeObserver} from "usehooks-ts"
 
 import {useRunId} from "@/oss/contexts/RunIdContext"
@@ -15,10 +15,7 @@ import {urlStateAtom} from "../../state/urlState"
 import useExpandableComparisonDataSource from "./hooks/useExpandableComparisonDataSource"
 import useScrollToScenario from "./hooks/useScrollToScenario"
 
-const EnhancedTable = dynamic(() => import("@/oss/components/EnhancedUIs/Table"), {
-    ssr: false,
-    loading: () => <EvalRunTestCaseTableSkeleton />,
-})
+import EnhancedTable from "@/oss/components/EnhancedUIs/Table"
 
 export const expendedRowAtom = atom<Record<string, boolean>>({})
 
@@ -90,7 +87,7 @@ const ComparisonTable = () => {
         )
     }
 
-    if (loading) {
+    if (loading || !EnhancedTable) {
         return <EvalRunTestCaseTableSkeleton />
     }
 

web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ScenarioTable.tsx

@@ -9,11 +9,8 @@ import {EvalRunTestCaseTableSkeleton} from "../../AutoEvalRun/components/EvalRun
 
 import useScrollToScenario from "./hooks/useScrollToScenario"
 import useTableDataSource from "./hooks/useTableDataSource"
+import EnhancedTable from "@/oss/components/EnhancedUIs/Table"
 
-const EnhancedTable = dynamic(() => import("@/oss/components/EnhancedUIs/Table"), {
-    ssr: false,
-    loading: () => <EvalRunTestCaseTableSkeleton />,
-})
 const VirtualizedScenarioTableAnnotateDrawer = dynamic(
     () => import("./assets/VirtualizedScenarioTableAnnotateDrawer"),
     {ssr: false},
@@ -34,33 +31,33 @@ const ScenarioTable = ({runId: propRunId}: {runId?: string}) => {
         box: "border-box",
     })
 
+    if (isLoadingSteps || !EnhancedTable) {
+        return <EvalRunTestCaseTableSkeleton />
+    }
+
     return (
         <div ref={tableContainerRef} className="grow flex flex-col w-full min-h-0">
-            {isLoadingSteps ? (
-                <EvalRunTestCaseTableSkeleton />
-            ) : (
-                <div className="relative w-full flex-1 min-h-0">
-                    {!scrollY ? null : (
-                        <EnhancedTable
-                            uniqueKey="scenario-table"
-                            columns={antColumns as any}
-                            dataSource={rows}
-                            scroll={{x: "max-content", y: scrollY - 45}}
-                            size="small"
-                            virtualized
-                            rowKey={(record: any) => record.key || record.scenarioId}
-                            className="agenta-scenario-table"
-                            rowClassName="scenario-row"
-                            tableLayout="fixed"
-                            skeletonRowCount={0}
-                            loading={false}
-                            ref={tableInstance}
-                        />
-                    )}
+            <div className="relative w-full flex-1 min-h-0">
+                {!scrollY ? null : (
+                    <EnhancedTable
+                        uniqueKey="scenario-table"
+                        columns={antColumns as any}
+                        dataSource={rows}
+                        scroll={{x: "max-content", y: scrollY - 45}}
+                        size="small"
+                        virtualized
+                        rowKey={(record: any) => record.key || record.scenarioId}
+                        className="agenta-scenario-table"
+                        rowClassName="scenario-row"
+                        tableLayout="fixed"
+                        skeletonRowCount={0}
+                        loading={false}
+                        ref={tableInstance}
+                    />
+                )}
 
-                    <VirtualizedScenarioTableAnnotateDrawer runId={runId} />
-                </div>
-            )}
+                <VirtualizedScenarioTableAnnotateDrawer runId={runId} />
+            </div>
         </div>
     )
 }

web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/MetricCell.tsx

@@ -81,7 +81,6 @@ const MetricCell = memo<MetricCellProps>(
 
         // Non-numeric arrays rendered as Tag list
         let formatted: ReactNode = formatMetricValue(metricKey, value)
-
         if (metricType === "boolean" && Array.isArray(value as any)) {
             const trueEntry = (distInfo as any).frequency.find((f: any) => f.value === true)
             const total = (distInfo as any).count ?? 0