diff --git a/.husky/pre-commit b/.husky/pre-commit new file mode 100755 index 0000000000..08dfe5c787 --- /dev/null +++ b/.husky/pre-commit @@ -0,0 +1,15 @@ +#!/usr/bin/env sh + +# Delegate to pre-commit which runs configured hooks (e.g. gitleaks) from .pre-commit-config.yaml +if command -v pre-commit >/dev/null 2>&1; then + pre-commit run --hook-stage pre-commit +else + echo "pre-commit not found. Install with one of the following:" + echo " pip install pre-commit # cross-platform" + echo " brew install pre-commit # macOS/Linux with Homebrew" + echo "" + echo "gitleaks is also required. Install with:" + echo " brew install gitleaks # macOS/Linux with Homebrew" + echo " or see https://github.com/gitleaks/gitleaks#installation for other options" + exit 1 +fi \ No newline at end of file diff --git a/.husky/pre-push b/.husky/pre-push new file mode 100755 index 0000000000..8aa51d15bc --- /dev/null +++ b/.husky/pre-push @@ -0,0 +1,12 @@ +#!/usr/bin/env sh + +# Delegate to pre-commit which runs pre-push hooks (e.g. gitleaks) from .pre-commit-config.yaml +if command -v pre-commit >/dev/null 2>&1; then + pre-commit run --hook-stage pre-push +else + echo "pre-commit not found. Install with one of the following methods:" + echo " pip install pre-commit gitleaks" + echo " brew install pre-commit gitleaks" + echo " (see https://pre-commit.com/#installation and https://github.com/gitleaks/gitleaks#installation for more options)" + exit 1 +fi diff --git a/api/oss/src/core/services/v0.py b/api/oss/src/core/services/v0.py index 0c25956a28..06597bd632 100644 --- a/api/oss/src/core/services/v0.py +++ b/api/oss/src/core/services/v0.py @@ -91,17 +91,27 @@ def _format_with_template( elif format == "curly": import re + # Extract variables that exist in the original template before replacement + # This allows us to distinguish template variables from {{}} in user input values + original_variables = set(re.findall(r"\{\{(.*?)\}\}", content)) + result = content for key, value in kwargs.items(): pattern = r"\{\{" + re.escape(key) + r"\}\}" old_result = result - result = re.sub(pattern, str(value), result) + # Escape backslashes in the replacement string to prevent regex interpretation + escaped_value = str(value).replace("\\", "\\\\") + result = re.sub(pattern, escaped_value, result) + + # Only check if ORIGINAL template variables remain unreplaced + # Don't error on {{}} that came from user input values + unreplaced_matches = set(re.findall(r"\{\{(.*?)\}\}", result)) + truly_unreplaced = original_variables & unreplaced_matches - unreplaced_matches = re.findall(r"\{\{(.*?)\}\}", result) - if unreplaced_matches: - log.info(f"WORKFLOW Found unreplaced variables: {unreplaced_matches}") + if truly_unreplaced: + log.info(f"WORKFLOW Found unreplaced variables: {truly_unreplaced}") raise ValueError( - f"Template variables not found in inputs: {', '.join(unreplaced_matches)}" + f"Template variables not found in inputs: {', '.join(sorted(truly_unreplaced))}" ) return result diff --git a/api/oss/src/dbs/postgres/tracing/utils.py b/api/oss/src/dbs/postgres/tracing/utils.py index 1dead72f74..bf65f645b4 100644 --- a/api/oss/src/dbs/postgres/tracing/utils.py +++ b/api/oss/src/dbs/postgres/tracing/utils.py @@ -1935,7 +1935,11 @@ def compute_cqvs( for k, v in PSC_ITEMS: if v[0] in pcts and v[1] in pcts: - pscs[k] = (pcts[v[1]] - pcts[v[0]]) / (pcts[v[1]] + pcts[v[0]]) + pscs[k] = ( + (pcts[v[1]] - pcts[v[0]]) / (pcts[v[1]] + pcts[v[0]]) + if (pcts[v[1]] + pcts[v[0]]) != 0 + else 0.0 + ) value["pscs"] = pscs @@ -1961,7 +1965,9 @@ def compute_pscs( for k, v in PSC_ITEMS: if v[0] in pcts and v[1] in pcts: - pscs[k] = (pcts[v[1]] - pcts[v[0]]) / pcts["p50"] + pscs[k] = ( + (pcts[v[1]] - pcts[v[0]]) / pcts["p50"] if pcts["p50"] != 0 else 0.0 + ) value["pscs"] = pscs diff --git a/api/oss/src/services/evaluators_service.py b/api/oss/src/services/evaluators_service.py index 83913ee43c..b0aedac68b 100644 --- a/api/oss/src/services/evaluators_service.py +++ b/api/oss/src/services/evaluators_service.py @@ -533,15 +533,26 @@ def _format_with_template( elif format == "curly": import re + # Extract variables that exist in the original template before replacement + # This allows us to distinguish template variables from {{}} in user input values + original_variables = set(re.findall(r"\{\{(.*?)\}\}", content)) + result = content for key, value in kwargs.items(): pattern = r"\{\{" + re.escape(key) + r"\}\}" old_result = result - result = re.sub(pattern, str(value), result) - unreplaced_matches = re.findall(r"\{\{(.*?)\}\}", result) - if unreplaced_matches: + # Escape backslashes in the replacement string to prevent regex interpretation + escaped_value = str(value).replace("\\", "\\\\") + result = re.sub(pattern, escaped_value, result) + + # Only check if ORIGINAL template variables remain unreplaced + # Don't error on {{}} that came from user input values + unreplaced_matches = set(re.findall(r"\{\{(.*?)\}\}", result)) + truly_unreplaced = original_variables & unreplaced_matches + + if truly_unreplaced: raise ValueError( - f"Template variables not found in inputs: {', '.join(unreplaced_matches)}" + f"Template variables not found in inputs: {', '.join(sorted(truly_unreplaced))}" ) return result diff --git a/api/pyproject.toml b/api/pyproject.toml index 241f316ffc..024944f631 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "api" -version = "0.59.3" +version = "0.59.4" description = "Agenta API" authors = [ { name = "Mahmoud Mabrouk", email = "mahmoud@agenta.ai" }, diff --git a/sdk/agenta/sdk/types.py b/sdk/agenta/sdk/types.py index 4440481bf5..448ac23b26 100644 --- a/sdk/agenta/sdk/types.py +++ b/sdk/agenta/sdk/types.py @@ -521,13 +521,26 @@ def _format_with_template(self, content: str, kwargs: Dict[str, Any]) -> str: elif self.template_format == "curly": import re + # Extract variables that exist in the original template before replacement + # This allows us to distinguish template variables from {{}} in user input values + original_variables = set(re.findall(r"\{\{(.*?)\}\}", content)) + result = content for key, value in kwargs.items(): - result = re.sub(r"\{\{" + key + r"\}\}", str(value), result) - if re.search(r"\{\{.*?\}\}", result): - unreplaced = re.findall(r"\{\{(.*?)\}\}", result) + # Escape backslashes in the replacement string to prevent regex interpretation + escaped_value = str(value).replace("\\", "\\\\") + result = re.sub( + r"\{\{" + re.escape(key) + r"\}\}", escaped_value, result + ) + + # Only check if ORIGINAL template variables remain unreplaced + # Don't error on {{}} that came from user input values + unreplaced_matches = set(re.findall(r"\{\{(.*?)\}\}", result)) + truly_unreplaced = original_variables & unreplaced_matches + + if truly_unreplaced: raise TemplateFormatError( - f"Unreplaced variables in curly template: {unreplaced}" + f"Unreplaced variables in curly template: {sorted(truly_unreplaced)}" ) return result else: diff --git a/sdk/pyproject.toml b/sdk/pyproject.toml index fd19935298..8d90fb82ce 100644 --- a/sdk/pyproject.toml +++ b/sdk/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "agenta" -version = "0.59.3" +version = "0.59.4" description = "The SDK for agenta is an open-source LLMOps platform." readme = "README.md" authors = [ diff --git a/web/.husky/install.mjs b/web/.husky/install.mjs new file mode 100644 index 0000000000..4fe00507c0 --- /dev/null +++ b/web/.husky/install.mjs @@ -0,0 +1,6 @@ +// Skip Husky install in production and CI +if (process.env.NODE_ENV === "production" || process.env.CI === "true") { + process.exit(0) +} +const husky = (await import("husky")).default +console.log(husky()) diff --git a/web/ee/package.json b/web/ee/package.json index 983b7d62b2..24065cd219 100644 --- a/web/ee/package.json +++ b/web/ee/package.json @@ -1,6 +1,6 @@ { "name": "@agenta/ee", - "version": "0.59.3", + "version": "0.59.4", "private": true, "engines": { "node": ">=18" diff --git a/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/index.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/index.tsx index e99df5017f..a5a90bc3d2 100644 --- a/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/index.tsx +++ b/web/ee/src/components/EvalRunDetails/AutoEvalRun/components/EvalRunFocusDrawer/assets/FocusDrawerContent/index.tsx @@ -4,7 +4,7 @@ import SimpleSharedEditor from "@agenta/oss/src/components/EditorViews/SimpleSha import VirtualizedSharedEditors from "@agenta/oss/src/components/EditorViews/VirtualizedSharedEditors" import {Collapse, CollapseProps, Tag, Tooltip} from "antd" import clsx from "clsx" -import {useAtomValue} from "jotai" +import {getDefaultStore, useAtomValue} from "jotai" import {loadable} from "jotai/utils" import {useRouter} from "next/router" @@ -17,7 +17,6 @@ import {urlStateAtom} from "@/oss/components/EvalRunDetails/state/urlState" import {formatMetricValue} from "@/oss/components/HumanEvaluations/assets/MetricDetailsPopover/assets/utils" import {getStatusLabel} from "@/oss/lib/constants/statusLabels" import { - evalAtomStore, scenarioStepFamily, evaluationRunStateFamily, } from "@/oss/lib/hooks/useEvaluationRunData/assets/atoms" @@ -519,7 +518,7 @@ const FocusDrawerContent = () => { // Helper: collect evaluator list for a run const getRunEvaluators = (rId: string) => { - const rState = evalAtomStore().get(evaluationRunStateFamily(rId)) + const rState = getDefaultStore().get(evaluationRunStateFamily(rId)) const evaluators = rState?.enrichedRun?.evaluators || [] return Array.isArray(evaluators) ? evaluators @@ -555,7 +554,7 @@ const FocusDrawerContent = () => { ) } - const metricData = evalAtomStore().get( + const metricData = getDefaultStore().get( runScopedMetricDataFamily({ runId: rId, scenarioId: scId, @@ -566,7 +565,7 @@ const FocusDrawerContent = () => { // Run-scoped error fallback let errorStep: any = null - const stepLoadableR = evalAtomStore().get( + const stepLoadableR = getDefaultStore().get( loadable(scenarioStepFamily({runId: rId, scenarioId: scId})), ) as any if (stepLoadableR?.state === "hasData") { @@ -793,7 +792,8 @@ const FocusDrawerContent = () => { }, ), children: Object.keys(metrics || {})?.map((metricKey) => { - const metricData = evalAtomStore().get( + + const metricData = getDefaultStore().get( runScopedMetricDataFamily({ runId: runId!, scenarioId: scenarioId!, @@ -802,10 +802,10 @@ const FocusDrawerContent = () => { }), ) - const errorStep = - !metricData?.distInfo || hasError - ? getErrorStep(`${evaluator.slug}.${metricKey}`, scenarioId) - : null + const errorStep = getErrorStep( + `${evaluator.slug}.${metricKey}`, + scenarioId, + ) let value if ( @@ -825,7 +825,6 @@ const FocusDrawerContent = () => { } const formatted = formatMetricValue(metricKey, value || "") - return (
{ matchedComparisonScenarios, baseRunId, invocationStep?.stepkey, + getErrorStep, ]) if (stepLoadable.state !== "hasData" || !enricedRun) { diff --git a/web/ee/src/components/EvalRunDetails/AutoEvalRun/index.tsx b/web/ee/src/components/EvalRunDetails/AutoEvalRun/index.tsx index 4896a13b70..e7c55ca326 100644 --- a/web/ee/src/components/EvalRunDetails/AutoEvalRun/index.tsx +++ b/web/ee/src/components/EvalRunDetails/AutoEvalRun/index.tsx @@ -31,7 +31,7 @@ const AutoEvalRunDetails = ({name, description, id, isLoading}: AutoEvalRunDetai return (
diff --git a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ComparisonScenarioTable.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ComparisonScenarioTable.tsx index d60ac8835e..11a935e276 100644 --- a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ComparisonScenarioTable.tsx +++ b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ComparisonScenarioTable.tsx @@ -3,7 +3,7 @@ import {RefObject, useEffect, useMemo} from "react" import {DownOutlined} from "@ant-design/icons" import clsx from "clsx" import {atom, useAtom, useAtomValue} from "jotai" -import dynamic from "next/dynamic" + import {useResizeObserver} from "usehooks-ts" import {useRunId} from "@/oss/contexts/RunIdContext" @@ -15,10 +15,7 @@ import {urlStateAtom} from "../../state/urlState" import useExpandableComparisonDataSource from "./hooks/useExpandableComparisonDataSource" import useScrollToScenario from "./hooks/useScrollToScenario" -const EnhancedTable = dynamic(() => import("@/oss/components/EnhancedUIs/Table"), { - ssr: false, - loading: () => , -}) +import EnhancedTable from "@/oss/components/EnhancedUIs/Table" export const expendedRowAtom = atom>({}) @@ -90,7 +87,7 @@ const ComparisonTable = () => { ) } - if (loading) { + if (loading || !EnhancedTable) { return } diff --git a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ScenarioTable.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ScenarioTable.tsx index c0219b19c8..dcc9b61896 100644 --- a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ScenarioTable.tsx +++ b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/ScenarioTable.tsx @@ -9,11 +9,8 @@ import {EvalRunTestCaseTableSkeleton} from "../../AutoEvalRun/components/EvalRun import useScrollToScenario from "./hooks/useScrollToScenario" import useTableDataSource from "./hooks/useTableDataSource" +import EnhancedTable from "@/oss/components/EnhancedUIs/Table" -const EnhancedTable = dynamic(() => import("@/oss/components/EnhancedUIs/Table"), { - ssr: false, - loading: () => , -}) const VirtualizedScenarioTableAnnotateDrawer = dynamic( () => import("./assets/VirtualizedScenarioTableAnnotateDrawer"), {ssr: false}, @@ -34,33 +31,33 @@ const ScenarioTable = ({runId: propRunId}: {runId?: string}) => { box: "border-box", }) + if (isLoadingSteps || !EnhancedTable) { + return + } + return (
- {isLoadingSteps ? ( - - ) : ( -
- {!scrollY ? null : ( - record.key || record.scenarioId} - className="agenta-scenario-table" - rowClassName="scenario-row" - tableLayout="fixed" - skeletonRowCount={0} - loading={false} - ref={tableInstance} - /> - )} +
+ {!scrollY ? null : ( + record.key || record.scenarioId} + className="agenta-scenario-table" + rowClassName="scenario-row" + tableLayout="fixed" + skeletonRowCount={0} + loading={false} + ref={tableInstance} + /> + )} - -
- )} + +
) } diff --git a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/MetricCell.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/MetricCell.tsx index 39b3631b99..0ffdddfc41 100644 --- a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/MetricCell.tsx +++ b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/MetricCell.tsx @@ -81,7 +81,6 @@ const MetricCell = memo( // Non-numeric arrays rendered as Tag list let formatted: ReactNode = formatMetricValue(metricKey, value) - if (metricType === "boolean" && Array.isArray(value as any)) { const trueEntry = (distInfo as any).frequency.find((f: any) => f.value === true) const total = (distInfo as any).count ?? 0 diff --git a/web/ee/src/components/EvalRunDetails/index.tsx b/web/ee/src/components/EvalRunDetails/index.tsx index fc97714bb4..61a61d1936 100644 --- a/web/ee/src/components/EvalRunDetails/index.tsx +++ b/web/ee/src/components/EvalRunDetails/index.tsx @@ -144,7 +144,7 @@ const EvaluationPage = memo(({evalType, runId}: {evalType: "auto" | "human"; run const breadcrumbs = useAtomValue(breadcrumbAtom, {store: rootStore}) const appendBreadcrumb = useSetAtom(appendBreadcrumbAtom, {store: rootStore}) const setEvalType = useSetAtom(setEvalTypeAtom) - const appId = useAppId() + const router = useRouter() const {isPreview, name, description, id} = useAtomValue( selectAtom( @@ -263,8 +263,9 @@ const EvaluationPage = memo(({evalType, runId}: {evalType: "auto" | "human"; run return (
{/** TODO: improve the component state specially AutoEvalRunDetails */} @@ -272,7 +273,7 @@ const EvaluationPage = memo(({evalType, runId}: {evalType: "auto" | "human"; run ) : isPreview && id ? ( @@ -281,14 +282,14 @@ const EvaluationPage = memo(({evalType, runId}: {evalType: "auto" | "human"; run ) : ( - + )}
) diff --git a/web/ee/src/components/EvaluationTable/ABTestingEvaluationTable.tsx b/web/ee/src/components/EvaluationTable/ABTestingEvaluationTable.tsx index ce5467f572..2552406e30 100644 --- a/web/ee/src/components/EvaluationTable/ABTestingEvaluationTable.tsx +++ b/web/ee/src/components/EvaluationTable/ABTestingEvaluationTable.tsx @@ -8,7 +8,7 @@ import {getDefaultStore, useAtomValue} from "jotai" import debounce from "lodash/debounce" import {useRouter} from "next/router" -import {useQueryParam} from "@/oss/hooks/useQuery" +import {useQueryParamState} from "@/oss/state/appState" import {EvaluationFlow} from "@/oss/lib/enums" import {exportABTestingEvaluationData} from "@/oss/lib/helpers/evaluate" import {isBaseResponse, isFuncResponse} from "@/oss/lib/helpers/playgroundResp" @@ -102,7 +102,22 @@ const ABTestingEvaluationTable: React.FC = ({ const [rows, setRows] = useState([]) const [, setEvaluationStatus] = useState(evaluation.status) - const [viewMode, setViewMode] = useQueryParam("viewMode", "card") + const [viewModeParam, setViewModeParam] = useQueryParamState("viewMode") + const viewMode = useMemo(() => { + if (Array.isArray(viewModeParam)) { + return viewModeParam[0] ?? "card" + } + if (typeof viewModeParam === "string" && viewModeParam) { + return viewModeParam + } + return "card" + }, [viewModeParam]) + const setViewMode = useCallback( + (nextMode: string) => { + setViewModeParam(nextMode, {method: "replace", shallow: true}) + }, + [setViewModeParam], + ) const {data: evaluationResults, mutate} = useEvaluationResults({ evaluationId: evaluation.id, onSuccess: () => { diff --git a/web/ee/src/components/EvaluationTable/SingleModelEvaluationTable.tsx b/web/ee/src/components/EvaluationTable/SingleModelEvaluationTable.tsx index 8273f6aa2e..9fde376831 100644 --- a/web/ee/src/components/EvaluationTable/SingleModelEvaluationTable.tsx +++ b/web/ee/src/components/EvaluationTable/SingleModelEvaluationTable.tsx @@ -20,7 +20,7 @@ import debounce from "lodash/debounce" import {useRouter} from "next/router" import SecondaryButton from "@/oss/components/SecondaryButton/SecondaryButton" -import {useQueryParam} from "@/oss/hooks/useQuery" +import {useQueryParamState} from "@/oss/state/appState" import {EvaluationFlow} from "@/oss/lib/enums" import {exportSingleModelEvaluationData} from "@/oss/lib/helpers/evaluate" import {isBaseResponse, isFuncResponse} from "@/oss/lib/helpers/playgroundResp" @@ -112,7 +112,22 @@ const SingleModelEvaluationTable: React.FC = ({ const [rows, setRows] = useState([]) const [evaluationStatus, setEvaluationStatus] = useState(evaluation.status) - const [viewMode, setViewMode] = useQueryParam("viewMode", "card") + const [viewModeParam, setViewModeParam] = useQueryParamState("viewMode") + const viewMode = useMemo(() => { + if (Array.isArray(viewModeParam)) { + return viewModeParam[0] ?? "card" + } + if (typeof viewModeParam === "string" && viewModeParam) { + return viewModeParam + } + return "card" + }, [viewModeParam]) + const setViewMode = useCallback( + (nextMode: string) => { + setViewModeParam(nextMode, {method: "replace", shallow: true}) + }, + [setViewModeParam], + ) const [accuracy, setAccuracy] = useState(0) const [isTestsetModalOpen, setIsTestsetModalOpen] = useState(false) diff --git a/web/ee/src/components/Evaluations/EvaluationCardView/index.tsx b/web/ee/src/components/Evaluations/EvaluationCardView/index.tsx index 96aef26f30..2b31bfb74e 100644 --- a/web/ee/src/components/Evaluations/EvaluationCardView/index.tsx +++ b/web/ee/src/components/Evaluations/EvaluationCardView/index.tsx @@ -15,7 +15,7 @@ import {useLocalStorage} from "usehooks-ts" import AlertPopup from "@/oss/components/AlertPopup/AlertPopup" import ParamsForm from "@/oss/components/ParamsForm" -import {useQueryParam} from "@/oss/hooks/useQuery" +import {useQueryParamState} from "@/oss/state/appState" import {EvaluationType} from "@/oss/lib/enums" import {testsetRowToChatMessages} from "@/oss/lib/helpers/testset" import useStatelessVariants from "@/oss/lib/hooks/useStatelessVariants" @@ -47,10 +47,29 @@ const EvaluationCardView: React.FC = ({ Record >("evaluationsState", {}) - const [scenarioId, setScenarioId] = useQueryParam( - "evaluationScenario", - evaluationsState[evaluation.id]?.lastVisitedScenario || evaluationScenarios[0]?.id || "", + const [scenarioParam, setScenarioParam] = useQueryParamState("evaluationScenario") + const fallbackScenarioId = useMemo(() => { + return ( + evaluationsState[evaluation.id]?.lastVisitedScenario || evaluationScenarios[0]?.id || "" + ) + }, [evaluation.id, evaluationScenarios, evaluationsState]) + const scenarioId = useMemo(() => { + if (Array.isArray(scenarioParam)) { + return scenarioParam[0] || fallbackScenarioId + } + if (typeof scenarioParam === "string" && scenarioParam) { + return scenarioParam + } + return fallbackScenarioId + }, [scenarioParam, fallbackScenarioId]) + const setScenarioId = useCallback( + (nextId: string) => { + if (!nextId) return + setScenarioParam(nextId, {method: "replace", shallow: true}) + }, + [setScenarioParam], ) + const [instructionsShown, setInstructionsShown] = useLocalStorage( "evalInstructionsShown", false, diff --git a/web/ee/src/lib/hooks/usePreviewEvaluations/assets/utils.ts b/web/ee/src/lib/hooks/usePreviewEvaluations/assets/utils.ts index e50327161f..61134926fc 100644 --- a/web/ee/src/lib/hooks/usePreviewEvaluations/assets/utils.ts +++ b/web/ee/src/lib/hooks/usePreviewEvaluations/assets/utils.ts @@ -301,7 +301,9 @@ export const enrichEvaluationRun = ({ const projectId = getProjectValues().projectId const baseVariants = filteredVariants.length ? filteredVariants : [] - const combinedVariants = (baseVariants.length ? baseVariants : fallbackVariants) as typeof fallbackVariants + const combinedVariants = ( + baseVariants.length ? baseVariants : fallbackVariants + ) as typeof fallbackVariants const normalizedVariants = combinedVariants .map((variant) => { @@ -374,8 +376,7 @@ export const enrichEvaluationRun = ({ finalAppId && (normalizedVariants.find((variant: any) => variant?.appId === finalAppId) || originalVariants.find( - (variant: any) => - variant?.appId === finalAppId || variant?.app_id === finalAppId, + (variant: any) => variant?.appId === finalAppId || variant?.app_id === finalAppId, )) const finalAppName = pickString( diff --git a/web/ee/src/services/observability/api/helper.ts b/web/ee/src/services/observability/api/helper.ts deleted file mode 100644 index 37f5e2353a..0000000000 --- a/web/ee/src/services/observability/api/helper.ts +++ /dev/null @@ -1,62 +0,0 @@ -import dayjs from "dayjs" - -import {GenerationDashboardData, TracingDashboardData} from "@/oss/lib/types_ee" - -export const normalizeDurationSeconds = (d = 0) => d / 1_000 - -export const formatTick = (ts: number | string, range: string) => - dayjs(ts).format(range === "24_hours" ? "h:mm a" : range === "7_days" ? "ddd" : "D MMM") - -export function tracingToGeneration( - tracing: TracingDashboardData, - range: string, -): GenerationDashboardData { - const buckets = tracing.buckets ?? [] - - let successCount = 0 - let errorCount = 0 - let totalCost = 0 - let totalTokens = 0 - let totalSuccessDuration = 0 - - const data = buckets.map((b) => { - const succC = b.total?.count ?? 0 - const errC = b.errors?.count ?? 0 - - const succCost = b.total?.costs ?? 0 - const errCost = b.errors?.costs ?? 0 - - const succTok = b.total?.tokens ?? 0 - const errTok = b.errors?.tokens ?? 0 - - const succDurS = normalizeDurationSeconds(b.total?.duration ?? 0) - - successCount += succC - errorCount += errC - totalCost += succCost + errCost - totalTokens += succTok + errTok - totalSuccessDuration += succDurS - - return { - timestamp: formatTick(b.timestamp, range), - success_count: succC, - failure_count: errC, - cost: succCost + errCost, - latency: succC ? succDurS / Math.max(succC, 1) : 0, // avg latency per success in the bucket - total_tokens: succTok + errTok, - } - }) - - const totalCount = successCount + errorCount - - return { - data, - total_count: totalCount, - failure_rate: totalCount ? errorCount / totalCount : 0, - total_cost: totalCost, - avg_cost: totalCount ? totalCost / totalCount : 0, - avg_latency: successCount ? totalSuccessDuration / successCount : 0, - total_tokens: totalTokens, - avg_tokens: totalCount ? totalTokens / totalCount : 0, - } -} diff --git a/web/ee/src/services/observability/api/index.ts b/web/ee/src/services/observability/api/index.ts deleted file mode 100644 index 2dabfe1c40..0000000000 --- a/web/ee/src/services/observability/api/index.ts +++ /dev/null @@ -1,168 +0,0 @@ -import axios from "@/oss/lib/api/assets/axiosConfig" -import {delay, pickRandom} from "@/oss/lib/helpers/utils" -import {GenericObject, WithPagination} from "@/oss/lib/Types" -import {Generation, GenerationDetails, Trace, TracingDashboardData} from "@/oss/lib/types_ee" -import {getProjectValues} from "@/oss/state/project" - -import {tracingToGeneration} from "./helper" -import {ObservabilityMock} from "./mock" - -//Prefix convention: -// - fetch: GET single entity from server -// - fetchAll: GET all entities from server -// - create: POST data to server -// - update: PUT data to server -// - delete: DELETE data from server - -const mock = false - -interface TableParams { - pagination?: { - page: number - pageSize: number - } - sorters?: GenericObject - filters?: GenericObject -} - -function tableParamsToApiParams(options?: Partial) { - const {page = 1, pageSize = 20} = options?.pagination || {} - const res: GenericObject = {page, pageSize} - if (options?.sorters) { - Object.entries(options.sorters).forEach( - ([key, val]) => (res[key] = val === "ascend" ? "asc" : "desc"), - ) - } - if (options?.filters) { - Object.entries(options.filters).forEach(([key, val]) => (res[key] = val)) - } - return res -} - -const generations = pickRandom(ObservabilityMock.generations, 100).map((item, ix) => ({ - ...item, - id: ix + 1 + "", -})) - -export const fetchAllGenerations = async (appId: string, options?: Partial) => { - const {projectId} = getProjectValues() - - const params = tableParamsToApiParams(options) - if (mock) { - const {page, pageSize} = params - await delay(800) - return { - data: generations.slice((page - 1) * pageSize, page * pageSize), - total: generations.length, - page, - pageSize, - } as WithPagination - } - - const response = await axios.get(`/observability/spans?project_id=${projectId}`, { - params: {app_id: appId, type: "generation", ...params}, - }) - return response.data as WithPagination -} - -export const fetchGeneration = async (generationId: string) => { - const {projectId} = getProjectValues() - - if (mock) { - await delay(800) - const generation = generations.find((item) => item.id === generationId) - if (!generation) throw new Error("not found!") - - return { - ...generation, - ...ObservabilityMock.generationDetail, - } as GenerationDetails - } - - const response = await axios.get( - `/observability/spans/${generationId}?project_id=${projectId}`, - { - params: {type: "generation"}, - }, - ) - return response.data as GenerationDetails -} - -export const fetchGenerationsDashboardData = async ( - appId: string | null | undefined, - _options: { - range: string - environment?: string - variant?: string - projectId?: string - signal?: AbortSignal - }, -) => { - const {projectId: propsProjectId, signal, ...options} = _options - const {projectId: _projectId} = getProjectValues() - const projectId = propsProjectId || _projectId - - const {range} = options - - if (signal?.aborted) { - throw new DOMException("Aborted", "AbortError") - } - - const responseTracing = await axios.post( - `/preview/tracing/spans/analytics?project_id=${projectId}`, - { - focus: "trace", - interval: 720, - filter: { - conditions: [ - { - field: "references", - operator: "in", - value: [ - { - id: appId, - }, - ], - }, - ], - }, - }, - ) - - const valTracing = responseTracing.data as TracingDashboardData - return tracingToGeneration(valTracing, range) -} - -export const deleteGeneration = async ( - generationIds: string[], - type = "generation", - ignoreAxiosError = true, -) => { - const {projectId} = getProjectValues() - - await axios.delete(`/observability/spans?project_id=${projectId}`, { - data: generationIds, - _ignoreError: ignoreAxiosError, - } as any) - return true -} - -export const fetchAllTraces = async (appId: string, options?: Partial) => { - const {projectId} = getProjectValues() - - const params = tableParamsToApiParams(options) - if (mock) { - const {page, pageSize} = params - await delay(800) - return { - data: generations.slice((page - 1) * pageSize, page * pageSize), - total: generations.length, - page, - pageSize, - } as WithPagination - } - const response = await axios.get(`/observability/traces?project_id=${projectId}`, { - params: {app_id: appId, type: "generation", ...params}, - }) - return response.data as WithPagination -} diff --git a/web/ee/src/services/observability/api/mock.ts b/web/ee/src/services/observability/api/mock.ts deleted file mode 100644 index a13e172d46..0000000000 --- a/web/ee/src/services/observability/api/mock.ts +++ /dev/null @@ -1,148 +0,0 @@ -import dayjs from "dayjs" - -import {randNum} from "@/oss/lib/helpers/utils" -import { - Generation, - GenerationKind, - GenerationDashboardData, - GenerationStatus, -} from "@/oss/lib/types_ee" - -const generations: Generation[] = [ - { - id: "1", - created_at: "2021-10-01T00:00:00Z", - variant: { - variant_id: "1", - variant_name: "default", - revision: 1, - }, - environment: "production", - status: GenerationStatus.OK, - spankind: GenerationKind.LLM, - metadata: { - cost: 0.0001, - latency: 0.32, - usage: { - total_tokens: 72, - prompt_tokens: 25, - completion_tokens: 47, - }, - }, - user_id: "u-8k3j4", - content: { - inputs: [ - {input_name: "country", input_value: "Pakistan"}, - {input_name: "criteria", input_value: "Most population"}, - ], - outputs: ["The most populous city in Pakistan is Karachi"], - internals: [], - }, - }, - { - id: "2", - created_at: "2023-10-01T00:00:00Z", - variant: { - variant_id: "2", - variant_name: "test", - revision: 1, - }, - environment: "staging", - status: GenerationStatus.ERROR, - spankind: GenerationKind.LLM, - metadata: { - cost: 0.0004, - latency: 0.845, - usage: { - total_tokens: 143, - prompt_tokens: 25, - completion_tokens: 118, - }, - }, - user_id: "u-8k3j4", - content: { - inputs: [], - outputs: [], - internals: [], - }, - }, - { - id: "3", - created_at: "2024-10-01T00:00:00Z", - variant: { - variant_id: "1", - variant_name: "default", - revision: 2, - }, - environment: "development", - status: GenerationStatus.OK, - spankind: GenerationKind.LLM, - metadata: { - cost: 0.0013, - latency: 0.205, - usage: { - total_tokens: 61, - prompt_tokens: 25, - completion_tokens: 36, - }, - }, - user_id: "u-7tij2", - content: { - inputs: [], - outputs: [], - internals: [], - }, - }, -] - -const generationDetail = { - content: { - inputs: [ - {input_name: "country", input_value: "Pakistan"}, - {input_name: "criteria", input_value: "Most population"}, - ], - outputs: ["The most populous city in Pakistan is Karachi"], - internals: [], - }, - config: { - system: "You are an expert in geography.", - user: "What is the city of {country} with the criteria {criteria}?", - variables: [ - {name: "country", type: "string"}, - {name: "criteria", type: "string"}, - ], - temperature: 0.7, - model: "gpt-3.5-turbo", - max_tokens: 100, - top_p: 0.9, - frequency_penalty: 0.5, - presence_penalty: 0, - }, -} - -const dashboardData = (count = 300): GenerationDashboardData["data"] => { - return Array(count) - .fill(true) - .map(() => { - const totalTokens = randNum(0, 600) - const promptTokens = randNum(0, 150) - return { - timestamp: randNum(dayjs().subtract(30, "days").valueOf(), dayjs().valueOf()), // b/w last 30 days - success_count: randNum(0, 20), - failure_count: randNum(0, 5), - latency: Math.random() * 1.5, - cost: Math.random() * 0.01, - total_tokens: totalTokens, - prompt_tokens: promptTokens, - completion_tokens: totalTokens - promptTokens, - enviornment: ["production", "staging", "development"][randNum(0, 2)], - variant: "default", - } - }) -} - -export const ObservabilityMock = { - generations, - generationDetail, - dashboardData, -} diff --git a/web/ee/src/state/observability/dashboard.ts b/web/ee/src/state/observability/dashboard.ts index a21cbd2950..e8c0215a71 100644 --- a/web/ee/src/state/observability/dashboard.ts +++ b/web/ee/src/state/observability/dashboard.ts @@ -3,9 +3,9 @@ import {eagerAtom} from "jotai-eager" import {atomWithQuery} from "jotai-tanstack-query" import {GenerationDashboardData} from "@/oss/lib/types_ee" -import {fetchGenerationsDashboardData} from "@/oss/services/observability/api" import {routerAppIdAtom} from "@/oss/state/app/atoms/fetcher" import {projectIdAtom} from "@/oss/state/project" +import {fetchGenerationsDashboardData} from "@/oss/services/tracing/api" const DEFAULT_RANGE = "30_days" diff --git a/web/oss/package.json b/web/oss/package.json index 2f5082d263..aba1172dbc 100644 --- a/web/oss/package.json +++ b/web/oss/package.json @@ -1,6 +1,6 @@ { "name": "@agenta/oss", - "version": "0.59.3", + "version": "0.59.4", "private": true, "engines": { "node": ">=18" diff --git a/web/oss/src/components/Playground/Components/Modals/CommitVariantChangesModal/assets/CommitVariantChangesModalContent/index.tsx b/web/oss/src/components/Playground/Components/Modals/CommitVariantChangesModal/assets/CommitVariantChangesModalContent/index.tsx index 1f9d87af9e..7114bd1f7d 100644 --- a/web/oss/src/components/Playground/Components/Modals/CommitVariantChangesModal/assets/CommitVariantChangesModalContent/index.tsx +++ b/web/oss/src/components/Playground/Components/Modals/CommitVariantChangesModal/assets/CommitVariantChangesModalContent/index.tsx @@ -157,6 +157,7 @@ const CommitVariantChangesModalContent = ({ placeholder="A unique variant name" className="w-full max-w-xs" value={selectedCommitType?.name} + disabled={selectedCommitType?.type !== "variant"} onChange={(e) => setSelectedCommitType((prev) => { const prevType = prev?.type diff --git a/web/oss/src/components/Playground/Components/PlaygroundTool/index.tsx b/web/oss/src/components/Playground/Components/PlaygroundTool/index.tsx index c190f64f9e..96b0a254d1 100644 --- a/web/oss/src/components/Playground/Components/PlaygroundTool/index.tsx +++ b/web/oss/src/components/Playground/Components/PlaygroundTool/index.tsx @@ -1,4 +1,4 @@ -import {useCallback, useEffect, useMemo, useRef, useState} from "react" +import React, {useCallback, useEffect, useMemo, useRef, useState} from "react" import {Input, Tooltip} from "antd" import clsx from "clsx" @@ -17,141 +17,358 @@ import SharedEditor from "../SharedEditor" import {TOOL_SCHEMA} from "./assets" -const PlaygroundTool = ({value, disabled, variantId, baseProperty, ...editorProps}) => { - const editorIdRef = useRef(uuidv4()) - const isReadOnly = Boolean(disabled) - const [minimized, setMinimized] = useState(false) - const [toolString, setToolString] = useState(() => { - try { - if (!value) { - return "" - } - return typeof value === "string" ? value : JSON5.stringify(value, null, 2) - } catch (e) { - return "" - } - }) - const [functionName, setFunctionName] = useState(() => { - try { - return typeof value === "string" - ? (JSON5.parse(value)?.function?.name ?? "") - : (value?.function?.name ?? "") - } catch (err) { - return "" - } - }) - const [functionDescription, setFunctionDescription] = useState(() => { - try { - return typeof value === "string" - ? (JSON5.parse(value)?.function?.description ?? "") - : (value?.function?.description ?? "") - } catch (err) { - return "" +export type ToolFunction = { + name?: string + description?: string + [k: string]: any +} + +export type ToolObj = { + function?: ToolFunction + [k: string]: any +} | null + +export interface PlaygroundToolProps { + value: unknown + disabled?: boolean + variantId: string + baseProperty?: {__id?: string} & Record + editorProps?: { + handleChange?: (obj: ToolObj) => void + } +} + +function safeStringify(obj: any): string { + try { + return JSON.stringify(obj, null, 2) + } catch { + return "" + } +} + +// stable stringify - sorts keys so deep equals is reliable +function stableStringify(input: any): string { + const seen = new WeakSet() + function sortKeys(value: any): any { + if (value && typeof value === "object") { + if (seen.has(value)) return null // guard against cycles + seen.add(value) + if (Array.isArray(value)) return value.map(sortKeys) + const out: Record = {} + Object.keys(value) + .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)) + .forEach((k) => { + out[k] = sortKeys(value[k]) + }) + return out } - }) - - const parsed = useMemo(() => { - if (!toolString) return null - try { - return JSON5.parse(toolString) - } catch (e) { - return null + return value + } + try { + return JSON.stringify(sortKeys(input)) + } catch { + return "" + } +} + +function deepEqual(a: any, b: any): boolean { + return stableStringify(a) === stableStringify(b) +} + +function toToolObj(value: unknown): ToolObj { + try { + if (typeof value === "string") return value ? (JSON5.parse(value) as ToolObj) : {} + if (value && typeof value === "object") return value as ToolObj + return {} + } catch { + // Keep last known good + return {} + } +} + +/** + * Two-way state that keeps editor JSON, parsed object, and header inputs in sync. + * - Debounces write backs into the editor to avoid stealing focus while typing in headers + * - Emits onChange only when the canonical object actually changes (deep compare) + * - Reacts to external prop changes to `initialValue` + */ +function useTwoWayToolState( + initialValue: unknown, + isReadOnly: boolean, + onChange?: (obj: ToolObj) => void, +) { + const [toolObj, setToolObj] = useState(() => toToolObj(initialValue)) + const [editorText, setEditorText] = useState(() => safeStringify(toolObj ?? {})) + const [editorValid, setEditorValid] = useState(true) + + // Header drafts and focus guards + const [isEditingName, setIsEditingName] = useState(false) + const [isEditingDesc, setIsEditingDesc] = useState(false) + const [nameDraft, setNameDraft] = useState(() => toolObj?.function?.name ?? "") + const [descDraft, setDescDraft] = useState(() => toolObj?.function?.description ?? "") + + // Last sent payload to avoid duplicate onChange calls + const lastSentSerializedRef = useRef(stableStringify(toolObj)) + + // Keep drafts synced when not being edited + useEffect(() => { + if (!isEditingName) setNameDraft(toolObj?.function?.name ?? "") + if (!isEditingDesc) setDescDraft(toolObj?.function?.description ?? "") + }, [toolObj, isEditingName, isEditingDesc]) + + // Emit to parent when canonical state changes + useEffect(() => { + if (isReadOnly || !onChange) return + const current = stableStringify(toolObj) + if (current !== lastSentSerializedRef.current) { + lastSentSerializedRef.current = current + onChange(toolObj) } - }, [toolString]) + }, [toolObj, onChange, isReadOnly]) - const syncToolFunctionField = useCallback( - (updater: (toolFunction: any) => any) => { - if (isReadOnly) return + // Debounced serializer for pushing toolObj changes back into editorText + const serTimerRef = useRef | null>(null) + const scheduleSerialize = useCallback((nextObj: ToolObj) => { + // Immediate update to avoid dropped first-change issues + setEditorText(safeStringify(nextObj ?? {})) + setEditorValid(true) + }, []) - setToolString((currentString) => { - if (!currentString) return currentString - - try { - const parsedTool = JSON5.parse(currentString) - const currentFunction = parsedTool.function ?? {} - const nextFunction = updater(currentFunction) - const isSameReference = - (parsedTool.function && nextFunction === parsedTool.function) || - nextFunction === currentFunction - const nextFunctionIsObject = - typeof nextFunction === "object" && nextFunction !== null - const isEmptyNoop = - !parsedTool.function && - nextFunction === currentFunction && - (!nextFunctionIsObject || Object.keys(nextFunction).length === 0) - - if (isSameReference || isEmptyNoop) { - return currentString - } + useEffect( + () => () => { + if (serTimerRef.current) clearTimeout(serTimerRef.current) + }, + [], + ) - const updatedTool = { - ...parsedTool, - function: nextFunction, - } + // React to external initialValue changes + const lastPropValueRef = useRef(stableStringify(toToolObj(initialValue))) + useEffect(() => { + const nextParsed = toToolObj(initialValue) + const nextSerialized = stableStringify(nextParsed) + if (nextSerialized !== lastPropValueRef.current) { + lastPropValueRef.current = nextSerialized + setToolObj(nextParsed) + setEditorText(safeStringify(nextParsed ?? {})) + setEditorValid(true) + if (!isEditingName) setNameDraft(nextParsed?.function?.name ?? "") + if (!isEditingDesc) setDescDraft(nextParsed?.function?.description ?? "") + } + }, [initialValue]) - const nextString = JSON.stringify(updatedTool, null, 2) - return nextString === currentString ? currentString : nextString - } catch (err) { - console.error(err) - return currentString + const onEditorChange = useCallback( + (text: string) => { + if (isReadOnly) return + setEditorText(text) + try { + const parsed = text ? (JSON5.parse(text) as ToolObj) : {} + setEditorValid(true) + setToolObj((prev) => (deepEqual(prev, parsed) ? prev : parsed)) + if (!isEditingName) setNameDraft(parsed?.function?.name ?? "") + if (!isEditingDesc) setDescDraft(parsed?.function?.description ?? "") + } catch { + setEditorValid(false) + } + }, + [isReadOnly, isEditingName, isEditingDesc], + ) + + const setFunctionName = useCallback( + (nextName: string) => { + if (isReadOnly) return + setNameDraft(nextName) + setToolObj((prev) => { + const base = prev ?? {} + const nextObj = { + ...base, + function: { + ...(base.function ?? {}), + name: nextName, + }, } + scheduleSerialize(nextObj) + return nextObj }) }, - [isReadOnly], + [isReadOnly, scheduleSerialize], ) - useEffect(() => { - if (isReadOnly) return - try { - const parsedTool = JSON5.parse(toolString) - editorProps?.handleChange?.(parsedTool) - setFunctionName((currentName) => { - const nextName = parsedTool?.function?.name ?? "" - if (currentName !== nextName) { - return nextName + const setFunctionDescription = useCallback( + (nextDesc: string) => { + if (isReadOnly) return + setDescDraft(nextDesc) + setToolObj((prev) => { + const base = prev ?? {} + const nextObj = { + ...base, + function: { + ...(base.function ?? {}), + description: nextDesc, + }, } - return currentName + scheduleSerialize(nextObj) + return nextObj }) - setFunctionDescription((currentDescription) => { - const nextDescription = parsedTool?.function?.description ?? "" - if (currentDescription !== nextDescription) { - return nextDescription + }, + [isReadOnly, scheduleSerialize], + ) + + return { + toolObj, + editorText, + editorValid, + nameDraft, + descDraft, + setFunctionName, + setFunctionDescription, + onEditorChange, + setIsEditingName, + setIsEditingDesc, + } +} + +/** + * Header component - isolated to avoid re-creating callbacks unnecessarily + */ +function ToolHeader(props: { + idForActions: string + name: string + desc: string + editorValid: boolean + isReadOnly: boolean + minimized: boolean + onNameFocus: () => void + onNameBlur: () => void + onDescFocus: () => void + onDescBlur: () => void + onNameChange: (v: string) => void + onDescChange: (v: string) => void + onToggleMinimize: () => void + onDelete?: () => void +}) { + const { + name, + desc, + editorValid, + isReadOnly, + minimized, + onNameFocus, + onNameBlur, + onDescFocus, + onDescBlur, + onNameChange, + onDescChange, + onToggleMinimize, + onDelete, + } = props + + return ( +
+
+
+ + onNameChange(e.target.value)} + /> + + + + onDescChange(e.target.value)} + /> + +
+
+ + +
+ ) +} + +const PlaygroundTool: React.FC = ({ + value, + disabled, + variantId, + baseProperty, + editorProps, +}) => { + const editorIdRef = useRef(uuidv4()) + const isReadOnly = Boolean(disabled) + const [minimized, setMinimized] = useState(false) + + const { + toolObj, + editorText, + editorValid, + nameDraft, + descDraft, + setFunctionName, + setFunctionDescription, + onEditorChange, + setIsEditingName, + setIsEditingDesc, + } = useTwoWayToolState(value, isReadOnly, editorProps?.handleChange) + + useAtomValue(variantByRevisionIdAtomFamily(variantId)) const appUriInfo = useAtomValue(appUriInfoAtom) const setPrompts = useSetAtom( - useMemo(() => promptsAtomFamily(variantId), [variant, variantId, appUriInfo?.routePath]), + useMemo(() => promptsAtomFamily(variantId), [variantId, appUriInfo?.routePath]), ) + const deleteMessage = useCallback(() => { if (isReadOnly) return - if (!baseProperty?.__id) { - console.warn("Cannot delete tool: tool property ID not found") + const id = baseProperty?.__id + if (!id) { + console.warn("Cannot delete tool - missing tool property ID") return } setPrompts((prevPrompts: any[] = []) => { return prevPrompts.map((prompt: any) => { const toolsArr = prompt?.llmConfig?.tools?.value if (Array.isArray(toolsArr)) { - const updatedTools = toolsArr.filter( - (tool: any) => tool.__id !== baseProperty.__id, - ) + const updatedTools = toolsArr.filter((tool: any) => tool.__id !== id) if (updatedTools.length !== toolsArr.length) { return { ...prompt, llmConfig: { ...prompt.llmConfig, tools: { - ...prompt.llmConfig.tools, + ...prompt.llmConfig?.tools, value: updatedTools, }, }, @@ -161,14 +378,13 @@ const PlaygroundTool = ({value, disabled, variantId, baseProperty, ...editorProp return prompt }) }) - }, [isReadOnly, variantId, baseProperty?.__id, setPrompts]) + }, [isReadOnly, baseProperty?.__id, setPrompts]) + return ( _div]:!w-auto [&_>_div]:!grow !my-0", - { - "[_.agenta-shared-editor]:w-full": isReadOnly, - }, + {"[_.agenta-shared-editor]:w-full": isReadOnly}, )} > { if (isReadOnly) return - setToolString(e) + onEditorChange(e) }} syncWithInitialValueChanges editorType="border" className={clsx([ "mt-2", - { - "[&_.agenta-editor-wrapper]:h-[calc(8px+calc(3*19.88px))] [&_.agenta-editor-wrapper]:overflow-y-auto [&_.agenta-editor-wrapper]:!mb-0": - minimized, - "[&_.agenta-editor-wrapper]:h-fit": !minimized, - }, + minimized + ? "[&_.agenta-editor-wrapper]:h-[calc(8px+calc(3*19.88px))] [&_.agenta-editor-wrapper]:overflow-y-auto [&_.agenta-editor-wrapper]:!mb-0" + : "[&_.agenta-editor-wrapper]:h-fit", ])} state={isReadOnly ? "readOnly" : "filled"} header={ -
-
-
- - { - if (isReadOnly) return - - const nextName = e.target.value - setFunctionName(nextName) - syncToolFunctionField((toolFunction) => { - if (toolFunction?.name === nextName) { - return toolFunction - } - - return { - ...toolFunction, - name: nextName, - } - }) - }} - /> - - - { - if (isReadOnly) return - - const nextDescription = e.target.value - setFunctionDescription(nextDescription) - syncToolFunctionField((toolFunction) => { - if ( - toolFunction?.description === - nextDescription - ) { - return toolFunction - } - - return { - ...toolFunction, - description: nextDescription, - } - }) - }} - /> - -
-
- - { - setMinimized((current) => !current) - }, - } - } - hideMarkdownToggle={true} - /> -
+ setIsEditingName(true)} + onNameBlur={() => setIsEditingName(false)} + onDescFocus={() => setIsEditingDesc(true)} + onDescBlur={() => setIsEditingDesc(false)} + onNameChange={setFunctionName} + onDescChange={setFunctionDescription} + onToggleMinimize={() => setMinimized((v) => !v)} + onDelete={deleteMessage} + /> } /> diff --git a/web/oss/src/components/pages/observability/assets/ObservabilityHeader/index.tsx b/web/oss/src/components/pages/observability/assets/ObservabilityHeader/index.tsx index f1be7b7087..6c71a97bcb 100644 --- a/web/oss/src/components/pages/observability/assets/ObservabilityHeader/index.tsx +++ b/web/oss/src/components/pages/observability/assets/ObservabilityHeader/index.tsx @@ -267,7 +267,7 @@ const ObservabilityHeader = ({columns}: ObservabilityHeaderProps) => { /> )} { - const query = parse(window.location.search.replace("?", "")) - - //do not update query if the value is the same - let changed = false - for (const key in queryObj) { - if (query[key]?.toString() !== queryObj[key]?.toString()) { - changed = true - break - } - } - if (!changed) return - - const newQuery = { - ...query, - ...queryObj, - } - //delete keys with undefined values - Object.keys(newQuery).forEach((key) => { - if (newQuery[key] === undefined || newQuery[key] === "") { - delete newQuery[key] - } - }) - - router[method]( - { - pathname: window.location.pathname, - query: newQuery, - }, - undefined, - {scroll: false}, - ) +const normalizeValue = ( + value: string | string[] | number | boolean | null | undefined, +): string | string[] | undefined => { + if (value === null || value === undefined) return undefined + if (Array.isArray(value)) { + const normalizedArray = value + .map((item) => { + if (item === null || item === undefined) return undefined + const str = String(item) + return str === "" ? undefined : str + }) + .filter((item): item is string => item !== undefined) + + return normalizedArray.length > 0 ? normalizedArray : undefined } + + const normalized = String(value) + return normalized === "" ? undefined : normalized +} + +const valuesAreEqual = ( + current: string | string[] | undefined, + next: string | string[] | undefined, +) => { + if (current === next) return true + if (Array.isArray(current) && Array.isArray(next)) { + if (current.length !== next.length) return false + return current.every((value, index) => value === next[index]) + } + return !Array.isArray(current) && !Array.isArray(next) && current === next } export function useQuery( method: Method = "push", ): [ParsedUrlQuery, (query: ParsedUrlQuery) => void] { - const router = useRouter() - const {query} = router + const query = useAppQuery() + const navigation = useAppNavigation() + const queryRef = useRef(query) + + useEffect(() => { + queryRef.current = query + }, [query]) - return [query, getUpdateQuery(router, method)] + const updateQuery = useCallback( + (queryObj: ParsedUrlQuery) => { + const nextQuery: Record = {} + let hasChanged = false + const currentQuery = queryRef.current + + Object.keys(queryObj).forEach((key) => { + const requestedValue = normalizeValue(queryObj[key] as any) + const currentValue = normalizeValue(currentQuery[key] as any) + + if (!valuesAreEqual(currentValue, requestedValue)) { + hasChanged = true + } + + nextQuery[key] = requestedValue + }) + + if (!hasChanged) return + + navigation.patchQuery(nextQuery, {method, shallow: true}) + }, + [method, navigation], + ) + + return [query, updateQuery] } export function useQueryParam( paramName: string, defaultValue?: string, method?: Method, -): [string, (val: string) => void] { +): [string | undefined, (val: string | undefined) => void] { const [query, updateQuery] = useQuery(method) - const value = (query as Record)[paramName] || defaultValue + const rawValue = query[paramName] + const value = Array.isArray(rawValue) ? rawValue[0] : (rawValue as string | undefined) - const setValue = (val: string) => { - updateQuery({[paramName]: val}) - } + const setValue = useCallback( + (val: string | undefined) => { + updateQuery({[paramName]: val}) + }, + [paramName, updateQuery], + ) - return [value, setValue] + return [value ?? defaultValue, setValue] } diff --git a/web/oss/src/lib/helpers/tracing.ts b/web/oss/src/lib/helpers/tracing.ts index 8b0340fa50..3bd829041e 100644 --- a/web/oss/src/lib/helpers/tracing.ts +++ b/web/oss/src/lib/helpers/tracing.ts @@ -12,7 +12,7 @@ * @param spans - Array of spans to sort * @returns New sorted array (does not mutate input) */ -export const sortSpansByStartTime = ( +export const sortSpansByStartTime = ( spans: T[], ): T[] => { return [...spans].sort((a, b) => { diff --git a/web/oss/src/services/observability/core/index.ts b/web/oss/src/services/observability/core/index.ts deleted file mode 100644 index ecd67e6ddd..0000000000 --- a/web/oss/src/services/observability/core/index.ts +++ /dev/null @@ -1,36 +0,0 @@ -import {fetchJson, getBaseUrl, ensureProjectId, ensureAppId} from "@/oss/lib/api/assets/fetchClient" - -//Prefix convention: -// - fetch: GET single entity from server -// - fetchAll: GET all entities from server -// - create: POST data to server -// - update: PUT data to server -// - delete: DELETE data from server - -export const fetchAllTraces = async (params = {}, appId: string) => { - const base = getBaseUrl() - const projectId = ensureProjectId() - const applicationId = ensureAppId(appId) - - const url = new URL(`${base}/observability/v1/traces`) - if (projectId) url.searchParams.set("project_id", projectId) - if (applicationId) url.searchParams.set("application_id", applicationId) - Object.entries(params).forEach(([key, value]) => { - if (value !== undefined && value !== null) { - url.searchParams.set(key, String(value)) - } - }) - - return fetchJson(url) -} - -export const deleteTrace = async (nodeId: string) => { - const base = getBaseUrl() - const projectId = ensureProjectId() - - const url = new URL(`${base}/observability/v1/traces`) - if (projectId) url.searchParams.set("project_id", projectId) - url.searchParams.set("node_id", nodeId) - - return fetchJson(url, {method: "DELETE"}) -} diff --git a/web/oss/src/services/observability/hooks/useTraces.ts b/web/oss/src/services/observability/hooks/useTraces.ts deleted file mode 100644 index 4b180580ab..0000000000 --- a/web/oss/src/services/observability/hooks/useTraces.ts +++ /dev/null @@ -1,147 +0,0 @@ -import {useEffect, useMemo} from "react" - -import {useAtomValue} from "jotai" -import useSWR, {preload} from "swr" - -import {buildNodeTree, observabilityTransformer} from "@/oss/lib/helpers/observability_helpers" -import {projectIdAtom} from "@/oss/state/project" - -import {fetchAllTraces} from "../core" -import {_AgentaRootsResponse, AgentaNodeDTO, AgentaRootsDTO, AgentaTreeDTO} from "../types" - -export const useTraces = ( - { - pagination, - sort, - filters, - traceTabs, - autoPrefetch, - waitUntil, - }: { - pagination: {size: number; page: number} - sort: {type: string; sorted: string; customRange?: {startTime: string; endTime: string}} - filters: any[] - traceTabs: string - autoPrefetch?: boolean - waitUntil?: boolean - }, - appId: string, -) => { - const queryParams = useMemo(() => { - const params: Record = { - size: pagination.size, - page: pagination.page, - focus: traceTabs === "chat" ? "node" : traceTabs, - } - - if (filters.length > 0) { - const sanitizedFilters = filters.map(({isPermanent, ...rest}) => rest) - - params.filtering = JSON.stringify({conditions: sanitizedFilters}) - } - - if (sort) { - if (sort.type === "standard") { - params.oldest = sort.sorted - } else if ( - sort.type === "custom" && - (sort.customRange?.startTime || sort.customRange?.endTime) - ) { - const {startTime, endTime} = sort.customRange - - if (startTime) params.oldest = startTime - if (endTime) params.newest = endTime - } - } - - return params - }, [traceTabs, pagination.size, pagination.page, filters, sort]) - - const fetcher = async () => { - const data = await fetchAllTraces(queryParams, appId) - - const transformedTraces: _AgentaRootsResponse[] = [] - - if (data?.roots) { - transformedTraces.push( - ...data.roots.flatMap((item: AgentaRootsDTO) => - observabilityTransformer(item.trees[0]), - ), - ) - } - - if (data?.trees) { - transformedTraces.push( - ...data.trees.flatMap((item: AgentaTreeDTO) => observabilityTransformer(item)), - ) - } - - if (data?.nodes) { - transformedTraces.push( - ...data.nodes - .flatMap((node: AgentaNodeDTO) => buildNodeTree(node)) - .flatMap((item: AgentaTreeDTO) => observabilityTransformer(item)), - ) - } - - return { - traces: transformedTraces, - traceCount: data?.count, - } - } - - const prefetchPage = async (pageNumber: number) => { - const nextParams = { - ...queryParams, - page: pageNumber, - } - const key = ["traces", appId, JSON.stringify(nextParams)] - await preload(key, async () => { - const data = await fetchAllTraces(nextParams, appId) - - const transformedTraces: _AgentaRootsResponse[] = [] - - if (data?.roots) { - transformedTraces.push( - ...data.roots.flatMap((item: AgentaRootsDTO) => - observabilityTransformer(item.trees[0]), - ), - ) - } - - if (data?.trees) { - transformedTraces.push( - ...data.trees.flatMap((item: AgentaTreeDTO) => observabilityTransformer(item)), - ) - } - - if (data?.nodes) { - transformedTraces.push( - ...data.nodes - .flatMap((node: AgentaNodeDTO) => buildNodeTree(node)) - .flatMap((item: AgentaTreeDTO) => observabilityTransformer(item)), - ) - } - - return { - traces: transformedTraces, - traceCount: data?.count, - } - }) - } - - useEffect(() => { - if (autoPrefetch && !waitUntil) { - prefetchPage(pagination.page + 1) - } - }, [autoPrefetch, pagination.page, JSON.stringify(queryParams), appId, waitUntil]) - - const projectId = useAtomValue(projectIdAtom) - const swrKey = !projectId || waitUntil ? null : ["traces", appId, JSON.stringify(queryParams)] - const swr = useSWR(swrKey, fetcher, { - revalidateOnFocus: false, - revalidateOnReconnect: false, - }) - - return swr -} diff --git a/web/oss/src/services/tracing/api/index.ts b/web/oss/src/services/tracing/api/index.ts index fa91ec861b..b5c61178dd 100644 --- a/web/oss/src/services/tracing/api/index.ts +++ b/web/oss/src/services/tracing/api/index.ts @@ -1,4 +1,7 @@ import {getBaseUrl, fetchJson, ensureProjectId, ensureAppId} from "@/oss/lib/api/assets/fetchClient" +import {getProjectValues} from "@/oss/state/project" +import {rangeToIntervalMinutes, tracingToGeneration} from "../lib/helpers" +import {GenerationDashboardData, TracingDashboardData} from "../types" export const fetchAllPreviewTraces = async (params: Record = {}, appId: string) => { const base = getBaseUrl() @@ -52,3 +55,69 @@ export const deletePreviewTrace = async (traceId: string) => { return fetchJson(url, {method: "DELETE"}) } + +export const fetchGenerationsDashboardData = async ( + appId: string | null | undefined, + _options: { + range: string + environment?: string + variant?: string + projectId?: string + signal?: AbortSignal + }, +): Promise => { + const {projectId: propsProjectId, signal, ...options} = _options + const {projectId: stateProjectId} = getProjectValues() + + const base = getBaseUrl() + const projectId = propsProjectId || stateProjectId + const applicationId = ensureAppId(appId || undefined) + + if (signal?.aborted) { + throw new DOMException("Aborted", "AbortError") + } + + const url = new URL(`${base}/preview/tracing/spans/analytics`) + if (projectId) url.searchParams.set("project_id", projectId) + if (applicationId) url.searchParams.set("application_id", applicationId) + + const conditions: any[] = [] + + if (applicationId) { + conditions.push({ + field: "references", + operator: "in", + value: [{id: applicationId}], + }) + } + if (options.environment) { + conditions.push({ + field: "environment", + operator: "eq", + value: options.environment, + }) + } + if (options.variant) { + conditions.push({ + field: "variant", + operator: "eq", + value: options.variant, + }) + } + + const payload: Record = { + focus: "trace", + interval: rangeToIntervalMinutes(options.range), + ...(conditions.length ? {filter: {conditions}} : {}), + } + + const response = await fetchJson(url, { + method: "POST", + headers: {"Content-Type": "application/json"}, + body: JSON.stringify(payload), + signal, + }) + + const valTracing = response as TracingDashboardData + return tracingToGeneration(valTracing, options.range) as GenerationDashboardData +} diff --git a/web/oss/src/services/tracing/lib/helpers.ts b/web/oss/src/services/tracing/lib/helpers.ts index 4cf9856eb0..8898c5b8bc 100644 --- a/web/oss/src/services/tracing/lib/helpers.ts +++ b/web/oss/src/services/tracing/lib/helpers.ts @@ -1,6 +1,13 @@ +import dayjs from "dayjs" import {sortSpansByStartTime} from "@/oss/lib/helpers/tracing" -import {TraceSpanNode, TracesResponse, SpansResponse} from "../types" +import { + TraceSpanNode, + TracesResponse, + SpansResponse, + TracingDashboardData, + GenerationDashboardData, +} from "../types" export const isTracesResponse = (data: any): data is TracesResponse => { return typeof data === "object" && "traces" in data @@ -52,3 +59,74 @@ export const transformTracingResponse = (data: TraceSpanNode[]): TraceSpanNode[] return data.map(enhance) } + +export const rangeToIntervalMinutes = (range: string): number => { + switch (range) { + case "1h": + return 60 + case "24h": + return 60 + case "7d": + return 360 + case "30d": + return 720 + default: + return 720 + } +} + +export const normalizeDurationSeconds = (d = 0) => d / 1_000 + +export const formatTick = (ts: number | string, range: string) => + dayjs(ts).format(range === "24_hours" ? "h:mm a" : range === "7_days" ? "ddd" : "D MMM") + +export function tracingToGeneration(tracing: TracingDashboardData, range: string) { + const buckets = tracing.buckets ?? [] + + let successCount = 0 + let errorCount = 0 + let totalCost = 0 + let totalTokens = 0 + let totalSuccessDuration = 0 + + const data = buckets.map((b) => { + const succC = b.total?.count ?? 0 + const errC = b.errors?.count ?? 0 + + const succCost = b.total?.costs ?? 0 + const errCost = b.errors?.costs ?? 0 + + const succTok = b.total?.tokens ?? 0 + const errTok = b.errors?.tokens ?? 0 + + const succDurS = normalizeDurationSeconds(b.total?.duration ?? 0) + + successCount += succC + errorCount += errC + totalCost += succCost + errCost + totalTokens += succTok + errTok + totalSuccessDuration += succDurS + + return { + timestamp: formatTick(b.timestamp, range), + success_count: succC, + failure_count: errC, + cost: succCost + errCost, + latency: succC ? succDurS / Math.max(succC, 1) : 0, // avg latency per success in the bucket + total_tokens: succTok + errTok, + } + }) + + const totalCount = successCount + errorCount + + return { + data, + total_count: totalCount, + failure_rate: totalCount ? errorCount / totalCount : 0, + total_cost: totalCost, + avg_cost: totalCount ? totalCost / totalCount : 0, + avg_latency: successCount ? totalSuccessDuration / successCount : 0, + total_tokens: totalTokens, + avg_tokens: totalCount ? totalTokens / totalCount : 0, + } +} diff --git a/web/oss/src/services/tracing/types/index.ts b/web/oss/src/services/tracing/types/index.ts index 01ea1a3a53..ecefd52148 100644 --- a/web/oss/src/services/tracing/types/index.ts +++ b/web/oss/src/services/tracing/types/index.ts @@ -119,3 +119,46 @@ export type SpansResponse = { count: number spans: TraceSpan[] } + +export interface TracingDashboardData { + buckets: { + errors: { + costs: number + count: number + duration: number + tokens: number + } + timestamp: string + total: { + costs: number + count: number + duration: number + tokens: number + } + window: number + }[] + count: number + version: string +} + +export interface GenerationDashboardData { + data: { + timestamp: number | string + success_count: number + failure_count: number + cost: number + latency: number + total_tokens: number + prompt_tokens: number + completion_tokens: number + enviornment: string + variant: string + }[] + total_count: number + failure_rate: number + total_cost: number + avg_cost: number + avg_latency: number + total_tokens: number + avg_tokens: number +} diff --git a/web/oss/src/state/newObservability/atoms/queries.ts b/web/oss/src/state/newObservability/atoms/queries.ts index 22bc5da09b..aacdf9b83b 100644 --- a/web/oss/src/state/newObservability/atoms/queries.ts +++ b/web/oss/src/state/newObservability/atoms/queries.ts @@ -221,7 +221,13 @@ export const tracesQueryAtom = atomWithInfiniteQuery((get) => { } if (field === "status_code" && value === "STATUS_CODE_OK") { - return {field, operator: "is_not", value: "STATUS_CODE_ERROR"} + if (operator === "is") { + return {field, operator: "is_not", value: "STATUS_CODE_ERROR"} + } + + if (operator === "is_not") { + return {field, operator: "is", value: "STATUS_CODE_ERROR"} + } } if (field.includes("annotation")) { diff --git a/web/oss/src/state/newObservability/utils/buildTraceQueryParams.ts b/web/oss/src/state/newObservability/utils/buildTraceQueryParams.ts index 4572de81c9..ed56345e1f 100644 --- a/web/oss/src/state/newObservability/utils/buildTraceQueryParams.ts +++ b/web/oss/src/state/newObservability/utils/buildTraceQueryParams.ts @@ -95,7 +95,13 @@ export const buildTraceQueryParams = ({ } if (field === "status_code" && value === "STATUS_CODE_OK") { - return {field, operator: "is_not", value: "STATUS_CODE_ERROR"} + if (operator === "is") { + return {field, operator: "is_not", value: "STATUS_CODE_ERROR"} + } + + if (operator === "is_not") { + return {field, operator: "is", value: "STATUS_CODE_ERROR"} + } } return {field, operator, value} diff --git a/web/package.json b/web/package.json index f5ad245cfa..282c47a75e 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "agenta-web", - "version": "0.59.3", + "version": "0.59.4", "workspaces": [ "ee", "oss",