{combinedMetricEntries.map(({fullKey, metric, evaluatorSlug, metricKey}, idx) => {
if (!metric || !Object.keys(metric || {}).length) return null
diff --git a/web/ee/src/components/EvalRunDetails/components/EvalRunScenarioNavigator/index.tsx b/web/ee/src/components/EvalRunDetails/components/EvalRunScenarioNavigator/index.tsx
index d768b25c9b..d6b32606aa 100644
--- a/web/ee/src/components/EvalRunDetails/components/EvalRunScenarioNavigator/index.tsx
+++ b/web/ee/src/components/EvalRunDetails/components/EvalRunScenarioNavigator/index.tsx
@@ -23,7 +23,6 @@ import {
displayedScenarioIdsFamily,
scenarioStepFamily,
} from "@/oss/lib/hooks/useEvaluationRunData/assets/atoms/runScopedScenarios"
-import {evalAtomStore as getGlobalStore} from "@/oss/lib/hooks/useEvaluationRunData/assets/atoms/store"
import {statusColorMap} from "../../HumanEvalRun/assets/helpers"
import EvalRunScenarioStatusTag from "../EvalRunScenarioStatusTag"
@@ -47,7 +46,6 @@ const EvalRunScenarioNavigator = ({
}) => {
const router = useRouter()
const runId = useRunId()
- const store = getGlobalStore()
// Get effective runId - use provided runId or fallback to current run context
const effectiveRunId = useMemo(() => {
@@ -62,11 +60,10 @@ const EvalRunScenarioNavigator = ({
// Get full scenario objects so we can access stable scenarioIndex
// Read from the same global store that writes are going to
- const allScenarios = useAtomValue(scenariosFamily(effectiveRunId), {store}) ?? []
+ const allScenarios = useAtomValue(scenariosFamily(effectiveRunId)) ?? []
// Get filtered scenario IDs from the displayedScenarioIdsFamily atom
- const filteredScenarioIds =
- useAtomValue(displayedScenarioIdsFamily(effectiveRunId), {store}) ?? []
+ const filteredScenarioIds = useAtomValue(displayedScenarioIdsFamily(effectiveRunId)) ?? []
// states for select dropdown
const [searchTerm, setSearchTerm] = useState("")
@@ -130,7 +127,6 @@ const EvalRunScenarioNavigator = ({
const {enqueueScenario} = useEvalScenarioQueue({concurrency: 5})
const status = useAtomValue(
useMemo(() => scenarioStatusAtomFamily(activeId), [activeId]),
- {store},
) as any
const rawStatus = status?.status
const isRunning = ["running", "EVALUATION_STARTED"].includes(rawStatus as string)
diff --git a/web/ee/src/components/EvalRunDetails/components/EvalRunScenarioStatusTag/index.tsx b/web/ee/src/components/EvalRunDetails/components/EvalRunScenarioStatusTag/index.tsx
index 16037b2e8e..365c6cbe47 100644
--- a/web/ee/src/components/EvalRunDetails/components/EvalRunScenarioStatusTag/index.tsx
+++ b/web/ee/src/components/EvalRunDetails/components/EvalRunScenarioStatusTag/index.tsx
@@ -6,10 +6,7 @@ import {useAtomValue} from "jotai"
import {loadable} from "jotai/utils"
import {getStatusLabel} from "@/oss/lib/constants/statusLabels"
-import {
- scenarioStatusFamily,
- evalAtomStore,
-} from "@/oss/lib/hooks/useEvaluationRunData/assets/atoms"
+import {scenarioStatusFamily} from "@/oss/lib/hooks/useEvaluationRunData/assets/atoms"
import {STATUS_COLOR, STATUS_COLOR_TEXT} from "./assets"
/**
@@ -35,8 +32,6 @@ const EvalRunScenarioStatusTag = ({
className,
showAsTag = true,
}: EvalRunScenarioStatusTagProps) => {
- const store = evalAtomStore()
-
/**
* Loadable atom wrapping scenarioStatusFamily, which provides the most
* up-to-date status for the given scenarioId. This can be either a status
@@ -47,7 +42,6 @@ const EvalRunScenarioStatusTag = ({
*/
const statusLoadable = useAtomValue(
useMemo(() => loadable(scenarioStatusFamily({scenarioId, runId})), [scenarioId, runId]),
- {store},
)
const scenarioStatus = statusLoadable.state === "hasData" ? statusLoadable.data : undefined
const status = (scenarioStatus?.status as string) || "pending"
diff --git a/web/ee/src/components/EvalRunDetails/components/EvalRunScenariosViewSelector/index.tsx b/web/ee/src/components/EvalRunDetails/components/EvalRunScenariosViewSelector/index.tsx
index d3f13ffab0..1604a482ac 100644
--- a/web/ee/src/components/EvalRunDetails/components/EvalRunScenariosViewSelector/index.tsx
+++ b/web/ee/src/components/EvalRunDetails/components/EvalRunScenariosViewSelector/index.tsx
@@ -11,13 +11,12 @@ import {runViewTypeAtom, urlStateAtom} from "../../state/urlState"
import {ENABLE_CARD_VIEW, VIEW_HUMAN_OPTIONS, VIEW_AUTO_OPTIONS} from "./assets/constants"
const EvalRunScenariosViewSelector = () => {
- const store = evalAtomStore()
const evalType = useAtomValue(evalTypeAtom)
// Read from the same global store that writes are going to
- const viewType = useAtomValue(runViewTypeAtom, {store})
+ const viewType = useAtomValue(runViewTypeAtom)
const [_isPending, startTransition] = useTransition()
- const setUrlState = useSetAtom(urlStateAtom, {store})
+ const setUrlState = useSetAtom(urlStateAtom)
// Sync local atom from urlStateAtom changes
return (
diff --git a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/ActionCell.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/ActionCell.tsx
index 62698519f0..17dcae39c4 100644
--- a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/ActionCell.tsx
+++ b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/ActionCell.tsx
@@ -1,17 +1,14 @@
import {memo, useCallback} from "react"
import {Button, Spin} from "antd"
-import {useAtomValue} from "jotai"
+import {getDefaultStore, useAtomValue} from "jotai"
import {loadable} from "jotai/utils"
import {useRunId} from "@/oss/contexts/RunIdContext"
import {virtualScenarioTableAnnotateDrawerAtom} from "@/oss/lib/atoms/virtualTable"
// Use EE run-scoped versions for multi-run support
-import {
- scenarioStepFamily,
- evalAtomStore,
-} from "../../../../../lib/hooks/useEvaluationRunData/assets/atoms"
+import {scenarioStepFamily} from "../../../../../lib/hooks/useEvaluationRunData/assets/atoms"
import RunEvalScenarioButton from "../../../HumanEvalRun/components/RunEvalScenarioButton"
import {CellWrapper} from "./CellComponents"
@@ -21,14 +18,13 @@ import {CellWrapper} from "./CellComponents"
* Shows either a "Run" button (if scenario hasn't been executed) or an "Annotate" action.
*/
const ActionCell = ({scenarioId, runId: propRunId}: {scenarioId: string; runId?: string}) => {
- const store = evalAtomStore()
+ const store = getDefaultStore()
const contextRunId = useRunId()
const effectiveRunId = propRunId || contextRunId
// Use global store for multi-run support
const stepLoadable = useAtomValue(
loadable(scenarioStepFamily({scenarioId, runId: effectiveRunId})),
- {store},
)
const openAnnotateDrawer = useCallback(() => {
diff --git a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/CellComponents.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/CellComponents.tsx
index cd2668a389..7caa706f20 100644
--- a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/CellComponents.tsx
+++ b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/CellComponents.tsx
@@ -70,8 +70,6 @@ export const InputCell = memo(
disableExpand?: boolean
runId?: string
}) => {
- const store = evalAtomStore()
-
// Use effective runId with proper fallback logic
const contextRunId = useRunId()
const effectiveRunId = useMemo(() => {
@@ -87,7 +85,6 @@ export const InputCell = memo(
effectiveRunId
? loadable(scenarioStepFamily({scenarioId, runId: effectiveRunId}))
: atom({state: "loading" as const}),
- {store},
)
if (stepLoadable.state !== "hasData" || !stepLoadable.data)
return (
diff --git a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/MetricCell.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/MetricCell.tsx
index 4328802974..39b3631b99 100644
--- a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/MetricCell.tsx
+++ b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/MetricCell/MetricCell.tsx
@@ -1,4 +1,4 @@
-import {type ReactNode, memo, useMemo} from "react"
+import {type ReactNode, memo, useCallback, useMemo} from "react"
import {Tag, Tooltip} from "antd"
import clsx from "clsx"
@@ -8,11 +8,12 @@ import {urlStateAtom} from "@/oss/components/EvalRunDetails/state/urlState"
import MetricDetailsPopover from "@/oss/components/HumanEvaluations/assets/MetricDetailsPopover" // adjust path if necessary
import {formatMetricValue} from "@/oss/components/HumanEvaluations/assets/MetricDetailsPopover/assets/utils" // same util used elsewhere
import {Expandable} from "@/oss/components/Tables/ExpandableCell"
-import {useRunId} from "@/oss/contexts/RunIdContext"
import {getStatusLabel} from "@/oss/lib/constants/statusLabels"
import {
- evalAtomStore,
+ evaluationRunStateFamily,
+ getCurrentRunId,
loadableScenarioStepFamily,
+ scenarioStepFamily,
} from "@/oss/lib/hooks/useEvaluationRunData/assets/atoms"
import {runScopedMetricDataFamily} from "@/oss/lib/hooks/useEvaluationRunData/assets/atoms/runScopedMetrics"
import {EvaluationStatus} from "@/oss/lib/Types"
@@ -21,6 +22,10 @@ import {STATUS_COLOR_TEXT} from "../../../EvalRunScenarioStatusTag/assets"
import {CellWrapper} from "../CellComponents" // CellWrapper is default export? need to check.
import {AnnotationValueCellProps, MetricCellProps, MetricValueCellProps} from "./types"
+import {AnnotationDto} from "@/oss/lib/hooks/useAnnotations/types"
+import {loadable, selectAtom} from "jotai/utils"
+import {UseEvaluationRunScenarioStepsFetcherResult} from "@/oss/lib/hooks/useEvaluationRunScenarioSteps/types"
+import deepEqual from "fast-deep-equal"
/*
* MetricCell – common renderer for metric columns (scenario-level or evaluator-level).
@@ -200,16 +205,12 @@ export const MetricValueCell = memo
(
[fallbackKey, metricKey, param, runId, scenarioId],
)
- const store = evalAtomStore()
-
const urlState = useAtomValue(urlStateAtom)
const isComparisonMode = Boolean(urlState.compare && urlState.compare.length > 0)
let value, distInfo
- const result = useAtomValue(runScopedMetricDataFamily(param as any), {store})
- const fallbackResult = useAtomValue(runScopedMetricDataFamily(fallbackParam as any), {
- store,
- })
+ const result = useAtomValue(runScopedMetricDataFamily(param as any))
+ const fallbackResult = useAtomValue(runScopedMetricDataFamily(fallbackParam as any))
value = result.value
distInfo = result.distInfo
@@ -287,6 +288,118 @@ export const MetricValueCell = memo(
)
// --- Annotation value cell -----------------------------------------------
+// It's a hot fix until we fix the backend issue for annotation metrics
+// In the backend the metrics endpoint is not returning all the type of annotation metrics
+const OUTPUTS_PREFIX = "data.outputs."
+const OUTPUT_SECTION_KEYS = ["metrics", "notes", "extra"] as const
+
+const getNestedValue = (source: any, path?: string): any => {
+ if (!source || !path) return undefined
+ const segments = path.split(".").filter(Boolean)
+ if (!segments.length) return undefined
+ return segments.reduce((acc, segment) => {
+ if (acc === undefined || acc === null) return undefined
+ return acc[segment]
+ }, source)
+}
+
+const normalizeOutputsPath = (path?: string): string | undefined => {
+ if (!path) return undefined
+ if (path.startsWith(OUTPUTS_PREFIX)) {
+ return path.slice(OUTPUTS_PREFIX.length)
+ }
+ if (path.startsWith("outputs.")) {
+ return path.slice("outputs.".length)
+ }
+ return path
+}
+
+// Extract annotation metric/notes/extra values straight from the invocation payload.
+const resolveAnnotationMetricValue = ({
+ annotations,
+ fieldPath,
+ metricKey,
+ name,
+}: {
+ annotations: AnnotationDto[]
+ fieldPath?: string
+ metricKey?: string
+ name?: string
+}) => {
+ if (!annotations?.length) return undefined
+
+ const fieldSegments = fieldPath?.split(".").filter(Boolean) ?? []
+
+ const annotationsBySlug = new Map()
+ annotations.forEach((ann) => {
+ const slug = ann?.references?.evaluator?.slug
+ if (slug) annotationsBySlug.set(slug, ann)
+ })
+
+ const slugIndex = fieldSegments.findIndex((segment) => annotationsBySlug.has(segment))
+ const slug = slugIndex >= 0 ? fieldSegments[slugIndex] : undefined
+ const remainderSegments = slugIndex >= 0 ? fieldSegments.slice(slugIndex + 1) : fieldSegments
+ const remainderPath = remainderSegments.length ? remainderSegments.join(".") : undefined
+
+ const keyCandidates = Array.from(
+ new Set(
+ [metricKey, name, remainderSegments.at(-1), fieldSegments.at(-1)]
+ .filter((key): key is string => Boolean(key))
+ .map((key) => key),
+ ),
+ )
+
+ const outputPathCandidates = Array.from(
+ new Set(
+ [
+ normalizeOutputsPath(fieldPath),
+ normalizeOutputsPath(remainderPath),
+ ...keyCandidates.flatMap((key) =>
+ OUTPUT_SECTION_KEYS.map((section) => `${section}.${key}`),
+ ),
+ ].filter((path): path is string => Boolean(path)),
+ ),
+ )
+
+ const rootPathCandidates = Array.from(
+ new Set([fieldPath, remainderPath, ...keyCandidates].filter(Boolean) as string[]),
+ )
+
+ const prioritizedAnnotations = (() => {
+ if (slug) {
+ const matched = annotationsBySlug.get(slug)
+ if (matched) return [matched]
+ }
+
+ const matchedByKey = annotations.filter((ann) => {
+ const outputs = ann?.data?.outputs
+ if (!outputs) return false
+ return keyCandidates.some((key) =>
+ OUTPUT_SECTION_KEYS.some(
+ (section) => getNestedValue(outputs[section], key) !== undefined,
+ ),
+ )
+ })
+
+ if (matchedByKey.length) return matchedByKey
+ return annotations
+ })()
+
+ for (const ann of prioritizedAnnotations) {
+ const outputs = ann?.data?.outputs ?? {}
+ for (const path of outputPathCandidates) {
+ const val = getNestedValue(outputs, path)
+ if (val !== undefined) return val
+ }
+
+ for (const path of rootPathCandidates) {
+ const val = getNestedValue(ann, path)
+ if (val !== undefined) return val
+ }
+ }
+
+ return undefined
+}
export const AnnotationValueCell = memo(
({
@@ -298,14 +411,153 @@ export const AnnotationValueCell = memo(
metricType,
fullKey,
distInfo: propsDistInfo,
+ runId,
}) => {
- const stepSlug = stepKey?.includes(".") ? stepKey.split(".")[1] : undefined
- const param = useMemo(
- () => ({scenarioId, stepSlug, metricKey: metricKey || ""}),
- [scenarioId, stepSlug, metricKey],
+ // Use effective runId with fallback using useMemo
+ const effectiveRunId = useMemo(() => {
+ if (runId) return runId
+ try {
+ return getCurrentRunId()
+ } catch (error) {
+ return ""
+ }
+ }, [runId])
+ // Get evaluators from run-scoped state instead of global atom
+ const evaluatorsSelector = useCallback((state: any) => {
+ return state?.enrichedRun?.evaluators ? Object.values(state.enrichedRun.evaluators) : []
+ }, [])
+ const evaluatorsAtom = useMemo(
+ () =>
+ selectAtom(evaluationRunStateFamily(effectiveRunId), evaluatorsSelector, deepEqual),
+ [effectiveRunId, evaluatorsSelector],
+ )
+ const evaluators = useAtomValue(evaluatorsAtom)
+ const stepDataLoadable = useAtomValue(
+ loadable(scenarioStepFamily({scenarioId, runId: effectiveRunId})),
+ )
+
+ let stepData: UseEvaluationRunScenarioStepsFetcherResult | undefined = undefined
+ if (stepDataLoadable.state === "hasData") {
+ stepData = stepDataLoadable.data
+ } else if (stepDataLoadable.state === "loading") {
+ // stepData = prevDataRef.current
+ }
+
+ // Memoize annotation steps for best performance (multi-step)
+ const _annotationSteps = useMemo(
+ () =>
+ (stepData?.annotationSteps ??
+ []) as UseEvaluationRunScenarioStepsFetcherResult["annotationSteps"],
+ [stepData],
+ )
+ // Build annotations per step key / slug
+ const annotationsByStep = useMemo(() => {
+ type AnnStep = UseEvaluationRunScenarioStepsFetcherResult["annotationSteps"][number]
+ const map: Record = {}
+ if (!_annotationSteps.length) return map
+
+ _annotationSteps.forEach((step) => {
+ const annotation = step.annotation
+ const fullKey = step.stepKey ?? (step as any).key
+ const evaluatorSlug = annotation?.references?.evaluator?.slug
+ const linkKeys = annotation?.links ? Object.keys(annotation.links) : []
+
+ const possibleKeys = new Set()
+ linkKeys.forEach((key) => {
+ if (key) possibleKeys.add(key)
+ })
+ if (fullKey) {
+ possibleKeys.add(fullKey)
+ const invocationKey = fullKey.includes(".") ? fullKey.split(".")[0] : fullKey
+ if (invocationKey) possibleKeys.add(invocationKey)
+ }
+ if (evaluatorSlug) {
+ possibleKeys.add(evaluatorSlug)
+ }
+
+ if (!possibleKeys.size) {
+ possibleKeys.add("__default__")
+ }
+
+ possibleKeys.forEach((key) => {
+ if (!map[key]) map[key] = []
+ map[key].push(step)
+ })
+ })
+ return map
+ }, [_annotationSteps])
+ const buildAnnotateData = useCallback(
+ (lookupKey?: string) => {
+ const fallbackSteps = _annotationSteps || []
+ const _steps = (lookupKey && annotationsByStep?.[lookupKey]) || fallbackSteps
+ const _annotations = _steps
+ .map((s) => s.annotation)
+ .filter(Boolean) as AnnotationDto[]
+ const annotationEvaluatorSlugs = _annotations
+ .map((annotation) => annotation?.references?.evaluator?.slug)
+ .filter(Boolean)
+
+ return {
+ annotations: _annotations,
+ evaluatorSlugs:
+ evaluators
+ ?.map((e) => e.slug)
+ .filter((slug) => !annotationEvaluatorSlugs.includes(slug)) || [],
+ evaluators:
+ evaluators?.filter((e) => !annotationEvaluatorSlugs.includes(e.slug)) || [],
+ }
+ },
+ [annotationsByStep, evaluators, _annotationSteps],
+ )
+ const annotationKeys = useMemo(
+ () => Object.keys(annotationsByStep).filter((key) => key !== "__default__"),
+ [annotationsByStep],
)
- const {value: metricVal, distInfo} = useAtomValue(metricDataFamily(param))
+ const resolvedAnnotationKey = useMemo(() => {
+ if (!annotationKeys.length) {
+ return annotationsByStep.__default__ ? "__default__" : undefined
+ }
+
+ if (stepKey && stepKey !== "metric") {
+ if (annotationsByStep[stepKey]) return stepKey
+ const suffixMatch = annotationKeys.find((key) => key.endsWith(stepKey))
+ if (suffixMatch) return suffixMatch
+ }
+
+ const slugCandidates = [metricKey, name, fieldPath]
+ .map((candidate) => candidate?.split(".")[0])
+ .filter((slug): slug is string => Boolean(slug))
+
+ for (const slug of slugCandidates) {
+ const slugMatch = annotationKeys.find(
+ (key) => key === slug || key.endsWith(`.${slug}`),
+ )
+ if (slugMatch) return slugMatch
+ }
+
+ if (annotationsByStep.__default__) {
+ return "__default__"
+ }
+
+ return annotationKeys[0]
+ }, [annotationKeys, annotationsByStep, fieldPath, metricKey, name, stepKey])
+ const annotationsForStep = useMemo(() => {
+ const annotateData = buildAnnotateData(resolvedAnnotationKey)
+ return annotateData.annotations
+ }, [buildAnnotateData, resolvedAnnotationKey])
+
+ const metricVal = useMemo(
+ () =>
+ resolveAnnotationMetricValue({
+ annotations: annotationsForStep,
+ fieldPath,
+ metricKey,
+ name,
+ }),
+ [annotationsForStep, fieldPath, metricKey, name],
+ )
+ const distInfo = propsDistInfo
return (
{
- const store = evalAtomStore()
+ const store = getDefaultStore()
// Annotate drawer state (global, per-run)
- const annotateDrawer = useAtomValue(virtualScenarioTableAnnotateDrawerAtom, {store})
+ const annotateDrawer = useAtomValue(virtualScenarioTableAnnotateDrawerAtom)
const setAnnotateDrawer = store.set
const scenarioId = annotateDrawer.scenarioId
diff --git a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/dataSourceBuilder.ts b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/dataSourceBuilder.ts
index a88821b936..beae4086d4 100644
--- a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/dataSourceBuilder.ts
+++ b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/dataSourceBuilder.ts
@@ -123,7 +123,7 @@ export function buildScenarioTableData({
...data,
name: metricName || data.name,
title: formattedMetricName,
- kind: "metric",
+ kind: "annotation",
path: data.name,
stepKey: "metric",
metricType: metricsFromEvaluators[evaluatorSlug]?.find(
diff --git a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/utils.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/utils.tsx
index de9d89d523..8c82d96fb2 100644
--- a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/utils.tsx
+++ b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/assets/utils.tsx
@@ -412,6 +412,7 @@ export function buildAntdColumns(
distInfo={distMap[c.path]}
stepKey={annotationStepKey}
name={c.name}
+ runId={effectiveRunId}
/>
)
}
diff --git a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useExpandableComparisonDataSource.tsx b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useExpandableComparisonDataSource.tsx
index 0908385f40..c4fd7933ab 100644
--- a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useExpandableComparisonDataSource.tsx
+++ b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useExpandableComparisonDataSource.tsx
@@ -1,12 +1,11 @@
import {useMemo} from "react"
import deepEqual from "fast-deep-equal"
-import {atom, useAtomValue} from "jotai"
+import {atom, getDefaultStore, useAtomValue} from "jotai"
import {atomFamily} from "jotai/utils"
import {filterColumns} from "@/oss/components/Filters/EditColumns/assets/helper"
import {
- evalAtomStore,
evaluationRunStateFamily,
runIndexFamily,
} from "@/oss/lib/hooks/useEvaluationRunData/assets/atoms"
@@ -157,20 +156,17 @@ const useExpandableComparisonDataSource = ({
baseRunId,
comparisonRunIds,
}: UseExpandableComparisonDataSourceProps) => {
- const store = evalAtomStore()
+ const store = getDefaultStore()
// const fetchMultipleRuns = useSetAtom(multiRunDataFetcherAtom)
- const comparisonRunsSteps = useAtomValue(comparisonRunsStepsAtom(comparisonRunIds), {store})
- const baseTestcases = useAtomValue(testcaseForScenarios(baseRunId), {store})
- const comparisonRunIndexes = useAtomValue(comparisonRunIndexesAtom(comparisonRunIds), {store})
+ const comparisonRunsSteps = useAtomValue(comparisonRunsStepsAtom(comparisonRunIds))
+ const baseTestcases = useAtomValue(testcaseForScenarios(baseRunId))
+ const comparisonRunIndexes = useAtomValue(comparisonRunIndexesAtom(comparisonRunIds))
- const comparisonRunsEvaluators = useAtomValue(comparisonRunsEvaluatorsAtom(comparisonRunIds), {
- store,
- })
+ const comparisonRunsEvaluators = useAtomValue(comparisonRunsEvaluatorsAtom(comparisonRunIds))
const metricsFromEvaluators = useAtomValue(
metricsFromEvaluatorsFamily([baseRunId, ...comparisonRunIds]),
- {store},
)
// Match scenarios by content rather than IDs
@@ -223,8 +219,8 @@ const useExpandableComparisonDataSource = ({
}, [baseTestcases, comparisonRunsSteps, comparisonRunIds.join(",")])
// Build columns using EXACT same approach as regular table (useTableDataSource)
- const runIndex = useAtomValue(runIndexFamily(baseRunId), {store})
- const evaluationRunState = useAtomValue(evaluationRunStateFamily(baseRunId), {store})
+ const runIndex = useAtomValue(runIndexFamily(baseRunId))
+ const evaluationRunState = useAtomValue(evaluationRunStateFamily(baseRunId))
const expendedRows = useAtomValue(expendedRowAtom)
const evaluators = evaluationRunState?.enrichedRun?.evaluators || []
const baseEvaluators = Array.isArray(evaluators) ? evaluators : Object.values(evaluators)
@@ -327,7 +323,7 @@ const useExpandableComparisonDataSource = ({
[columnsWithRunSpecificSteps, baseRunId, expendedRows],
)
- const hiddenColumns = useAtomValue(editColumnsFamily(baseRunId), {store})
+ const hiddenColumns = useAtomValue(editColumnsFamily(baseRunId))
const antColumns = useMemo(
() => filterColumns(baseAntColumns, hiddenColumns),
@@ -343,7 +339,7 @@ const useExpandableComparisonDataSource = ({
const loading = false
// Build rows with actual scenario data - use the SAME approach as regular table
- const scenarioIds = useAtomValue(displayedScenarioIdsFamily(baseRunId), {store}) || []
+ const scenarioIds = useAtomValue(displayedScenarioIdsFamily(baseRunId)) || []
const rows = useMemo(() => {
const builtRows = scenarioIds.map((scenarioId, idx) => {
diff --git a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useTableDataSource.ts b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useTableDataSource.ts
index 95f0a6f3ab..ad1db674ce 100644
--- a/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useTableDataSource.ts
+++ b/web/ee/src/components/EvalRunDetails/components/VirtualizedScenarioTable/hooks/useTableDataSource.ts
@@ -18,7 +18,6 @@ import {
displayedScenarioIdsFamily,
loadableScenarioStepFamily,
} from "../../../../../lib/hooks/useEvaluationRunData/assets/atoms/runScopedScenarios"
-import {evalAtomStore} from "../../../../../lib/hooks/useEvaluationRunData/assets/atoms/store"
import {buildScenarioTableData, buildScenarioTableRows} from "../assets/dataSourceBuilder"
import {buildAntdColumns} from "../assets/utils"
@@ -78,14 +77,13 @@ export const metricsFromEvaluatorsFamily = atomFamily(
const useTableDataSource = () => {
const runId = useRunId()
- const store = evalAtomStore()
// states
- const [editColumns, setEditColumns] = useAtom(editColumnsFamily(runId), {store})
+ const [editColumns, setEditColumns] = useAtom(editColumnsFamily(runId))
// Read from the same global store that writes are going to
- const scenarioIds = useAtomValue(displayedScenarioIdsFamily(runId), {store}) || EMPTY_SCENARIOS
- const allScenariosLoaded = useAtomValue(allScenariosLoadedFamily(runId), {store})
+ const scenarioIds = useAtomValue(displayedScenarioIdsFamily(runId)) || EMPTY_SCENARIOS
+ const allScenariosLoaded = useAtomValue(allScenariosLoadedFamily(runId))
// const metricDistributions = useAtomValue(runMetricsStatsAtom)
const runIndex = useAtomValue(runIndexFamily(runId))
@@ -93,7 +91,7 @@ const useTableDataSource = () => {
useAtomValue(metricsFromEvaluatorsFamily(runId)) || EMPTY_SCENARIOS
// temporary implementation to implement loading state for auto eval
const loadable = useAtomValue(loadableScenarioStepFamily({runId, scenarioId: scenarioIds?.[0]}))
- const evaluationRunState = useAtomValue(evaluationRunStateFamily(runId), {store})
+ const evaluationRunState = useAtomValue(evaluationRunStateFamily(runId))
const evaluators = evaluationRunState?.enrichedRun?.evaluators || []
const isLoadingSteps = useMemo(
diff --git a/web/ee/src/components/EvalRunDetails/index.tsx b/web/ee/src/components/EvalRunDetails/index.tsx
index accc7a632e..fc97714bb4 100644
--- a/web/ee/src/components/EvalRunDetails/index.tsx
+++ b/web/ee/src/components/EvalRunDetails/index.tsx
@@ -1,9 +1,9 @@
-import {memo, useCallback, useEffect, useMemo} from "react"
+import {memo, useCallback, useEffect} from "react"
import {Spin, Typography} from "antd"
import clsx from "clsx"
import deepEqual from "fast-deep-equal"
-import {createStore, getDefaultStore, Provider, useAtomValue, useSetAtom} from "jotai"
+import {getDefaultStore, useAtomValue, useSetAtom} from "jotai"
import {selectAtom} from "jotai/utils"
import {useRouter} from "next/router"
@@ -16,7 +16,6 @@ import {appendBreadcrumbAtom, breadcrumbAtom, setBreadcrumbsAtom} from "@/oss/li
import {isUuid} from "@/oss/lib/helpers/utils"
import useEvaluationRunData from "@/oss/lib/hooks/useEvaluationRunData"
import {
- evalAtomStore,
evaluationRunStateFamily,
initializeRun,
} from "@/oss/lib/hooks/useEvaluationRunData/assets/atoms"
diff --git a/web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/index.tsx b/web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/index.tsx
index 5a7c03a731..1dce19c1f0 100644
--- a/web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/index.tsx
+++ b/web/ee/src/components/HumanEvaluations/assets/MetricDetailsPopover/index.tsx
@@ -5,10 +5,7 @@ import clsx from "clsx"
import {useAtomValue} from "jotai"
import {Expandable} from "@/oss/components/Tables/ExpandableCell"
-import {
- evalAtomStore,
- runMetricsStatsCacheFamily,
-} from "@/oss/lib/hooks/useEvaluationRunData/assets/atoms"
+import {runMetricsStatsCacheFamily} from "@/oss/lib/hooks/useEvaluationRunData/assets/atoms"
import {EvaluatorDto} from "@/oss/lib/hooks/useEvaluators/types"
import {extractPrimitive, inferMetricType} from "@/oss/lib/metricUtils"
@@ -270,10 +267,8 @@ export const MetricDetailsPopoverWrapper = memo(
[evaluatorSlug, evaluatorMetricKey],
)
- const store = evalAtomStore()
-
// Use run-scoped stats cache instead of global cache
- const runStatsCache = useAtomValue(runMetricsStatsCacheFamily(runId), {store})
+ const runStatsCache = useAtomValue(runMetricsStatsCacheFamily(runId))
const stats = statsOverride ?? runStatsCache?.[metricKey]
const rawPrimitive = useMemo(() => extractPrimitive(stats), [stats])
diff --git a/web/ee/src/components/pages/evaluations/utils.ts b/web/ee/src/components/pages/evaluations/utils.ts
index af9d1c59ac..e8a12d42a8 100644
--- a/web/ee/src/components/pages/evaluations/utils.ts
+++ b/web/ee/src/components/pages/evaluations/utils.ts
@@ -2,6 +2,16 @@ import {EvaluationRow} from "@/oss/components/HumanEvaluations/types"
type Nullable = T | null | undefined
+const pickString = (...values: unknown[]): string | undefined => {
+ for (const value of values) {
+ if (typeof value !== "string") continue
+ const trimmed = value.trim()
+ if (trimmed.length === 0) continue
+ return trimmed
+ }
+ return undefined
+}
+
const parseInvocationMetadata = (
evaluation: EvaluationRow,
): {
@@ -12,7 +22,18 @@ const parseInvocationMetadata = (
revisionLabel?: string | number
} | null => {
const dataSteps: any[] = (evaluation as any)?.data?.steps || []
- const invocationStep = dataSteps.find((step) => step?.type === "invocation")
+ const invocationStep = dataSteps.find((step) => {
+ if (!step) return false
+ if (step?.type === "invocation") return true
+ const refs = step.references ?? step ?? {}
+ return Boolean(
+ refs?.application ||
+ refs?.applicationRevision ||
+ refs?.application_revision ||
+ refs?.applicationRef ||
+ refs?.application_ref,
+ )
+ })
if (!invocationStep) return null
const references = invocationStep.references ?? invocationStep ?? {}
@@ -23,58 +44,71 @@ const parseInvocationMetadata = (
applicationRevision?.application ||
references.applicationRef ||
references.application_ref
- const variantRef = references.variant || references.variantRef || references.variant_ref
-
- const rawAppId =
- applicationRef?.id ||
- applicationRef?.app_id ||
- applicationRef?.appId ||
- references.application?.id ||
- references.application?.app_id ||
- applicationRevision?.application_id ||
- applicationRevision?.applicationId
-
- const rawAppName =
- applicationRef?.name ||
- applicationRef?.slug ||
- references.application?.name ||
- references.application?.slug
-
- const rawVariantName =
- variantRef?.name ||
- variantRef?.slug ||
- variantRef?.variantName ||
- variantRef?.variant_name ||
- applicationRef?.name ||
- applicationRef?.slug ||
- references.application?.name ||
- references.application?.slug ||
- invocationStep.key
-
- const rawRevisionId =
- variantRef?.id ||
- variantRef?.revisionId ||
- variantRef?.revision_id ||
- applicationRevision?.id ||
- applicationRevision?.revisionId ||
- applicationRevision?.revision_id
+ const variantRef =
+ references.variant ||
+ references.variantRef ||
+ references.variant_ref ||
+ references.applicationVariant ||
+ references.application_variant ||
+ applicationRevision?.variant ||
+ applicationRef?.variant
+
+ const rawAppId = pickString(
+ applicationRef?.id,
+ applicationRef?.app_id,
+ applicationRef?.appId,
+ applicationRevision?.application_id,
+ applicationRevision?.applicationId,
+ references.application?.id,
+ references.application?.app_id,
+ references.application?.appId,
+ )
+
+ const rawAppName = pickString(
+ applicationRef?.name,
+ applicationRef?.slug,
+ references.application?.name,
+ references.application?.slug,
+ )
+
+ const rawVariantName = pickString(
+ variantRef?.name,
+ variantRef?.slug,
+ variantRef?.variantName,
+ variantRef?.variant_name,
+ applicationRef?.variantName,
+ applicationRef?.variant_name,
+ references.application?.variantName,
+ references.application?.variant_name,
+ invocationStep.key,
+ )
+
+ const rawRevisionId = pickString(
+ variantRef?.id,
+ variantRef?.revisionId,
+ variantRef?.revision_id,
+ applicationRevision?.id,
+ applicationRevision?.revisionId,
+ applicationRevision?.revision_id,
+ )
const revisionLabel =
- variantRef?.version ??
- variantRef?.revision ??
- variantRef?.revisionLabel ??
- applicationRevision?.revision ??
- applicationRevision?.version ??
- applicationRevision?.name ??
- null
+ pickString(
+ variantRef?.version,
+ variantRef?.revision,
+ variantRef?.revisionLabel,
+ applicationRevision?.revision,
+ applicationRevision?.version,
+ applicationRevision?.name,
+ ) ?? null
if (!rawAppId && !rawRevisionId && !rawVariantName) return null
return {
- appId: typeof rawAppId === "string" ? rawAppId : undefined,
- appName: typeof rawAppName === "string" ? rawAppName : undefined,
- revisionId: typeof rawRevisionId === "string" ? rawRevisionId : undefined,
- variantName: typeof rawVariantName === "string" ? rawVariantName : undefined,
+ appId: rawAppId,
+ appName: rawAppName,
+ revisionId: rawRevisionId,
+ variantName: rawVariantName,
revisionLabel: revisionLabel ?? undefined,
}
}
@@ -91,27 +125,39 @@ export const extractPrimaryInvocation = (
if (!evaluation) return null
const variants = (evaluation as any)?.variants
- if (Array.isArray(variants) && variants.length) {
- const variant = variants[0]
- return {
- appId:
- variant?.appId ||
- (typeof variant?.app_id === "string" ? variant.app_id : undefined) ||
- (typeof variant?.applicationId === "string" ? variant.applicationId : undefined),
- appName: (variant as any)?.appName || (variant as any)?.appSlug,
- revisionId:
- (variant as any)?.id ||
- (typeof variant?.revisionId === "string" ? variant.revisionId : undefined) ||
- (typeof variant?.revision_id === "string" ? variant.revision_id : undefined),
- variantName: variant?.variantName || variant?.name || (variant as any)?.slug,
- revisionLabel:
- (variant as any)?.revisionLabel ||
- (variant as any)?.revision ||
- (variant as any)?.version,
- }
+ const variant = Array.isArray(variants) && variants.length ? variants[0] : undefined
+ const metadataFromSteps = parseInvocationMetadata(evaluation)
+
+ if (!variant && !metadataFromSteps) return null
+
+ const variantDetails = variant
+ ? {
+ appId:
+ variant?.appId ||
+ (typeof variant?.app_id === "string" ? variant.app_id : undefined) ||
+ (typeof variant?.applicationId === "string" ? variant.applicationId : undefined),
+ appName: (variant as any)?.appName || (variant as any)?.appSlug,
+ revisionId:
+ (variant as any)?.id ||
+ (typeof variant?.revisionId === "string" ? variant.revisionId : undefined) ||
+ (typeof variant?.revision_id === "string" ? variant.revision_id : undefined),
+ variantName: variant?.variantName || variant?.name || (variant as any)?.slug,
+ revisionLabel:
+ (variant as any)?.revisionLabel ||
+ (variant as any)?.revision ||
+ (variant as any)?.version,
+ }
+ : undefined
+
+ const resolved = {
+ appId: metadataFromSteps?.appId ?? variantDetails?.appId,
+ appName: metadataFromSteps?.appName ?? variantDetails?.appName,
+ revisionId: variantDetails?.revisionId ?? metadataFromSteps?.revisionId,
+ variantName: variantDetails?.variantName ?? metadataFromSteps?.variantName,
+ revisionLabel: variantDetails?.revisionLabel ?? metadataFromSteps?.revisionLabel,
}
- return parseInvocationMetadata(evaluation)
+ return resolved
}
export const extractEvaluationAppId = (evaluation: EvaluationRow): string | undefined => {
diff --git a/web/ee/src/lib/helpers/serviceValidations.ts b/web/ee/src/lib/helpers/serviceValidations.ts
index 87849c1098..6e86ae9e39 100644
--- a/web/ee/src/lib/helpers/serviceValidations.ts
+++ b/web/ee/src/lib/helpers/serviceValidations.ts
@@ -1,17 +1,17 @@
-const UUID_V4_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i
-const MONGO_OBJECT_ID_RE = /^[0-9a-f]{24}$/i // keep if backend allows ObjectIds
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i
+const MONGO_OBJECT_ID_RE = /^[0-9a-f]{24}$/i
export function isValidId(id: unknown): id is string {
if (typeof id !== "string") return false
const s = id.trim()
if (!s) return false
if (s.includes("/") || s.includes("\\") || s.includes("..")) return false
- return UUID_V4_RE.test(s) || MONGO_OBJECT_ID_RE.test(s)
+ return UUID_RE.test(s) || MONGO_OBJECT_ID_RE.test(s)
}
export function assertValidId(id: unknown, label = "id"): string {
if (!isValidId(id)) {
- throw new TypeError(`Invalid ${label}: must be a UUID v4 or 24-hex ObjectId`)
+ throw new TypeError(`Invalid ${label}: must be a UUID or 24-hex ObjectId`)
}
return (id as string).trim()
}
diff --git a/web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/store.ts b/web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/store.ts
index bb2014fd41..00c6548473 100644
--- a/web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/store.ts
+++ b/web/ee/src/lib/hooks/useEvaluationRunData/assets/atoms/store.ts
@@ -22,7 +22,7 @@ function createGlobalStore() {
}
// Global singleton store that persists across HMR
-const globalStore: ReturnType =
+const globalStore: ReturnType =
(globalThis as any)[globalStoreKey] || createGlobalStore()
;(globalThis as any)[globalStoreKey] = globalStore
diff --git a/web/ee/src/lib/hooks/useEvaluationRunData/useEvalRunScenarioData.tsx b/web/ee/src/lib/hooks/useEvaluationRunData/useEvalRunScenarioData.tsx
index 85c5bf75c2..a1656f9783 100644
--- a/web/ee/src/lib/hooks/useEvaluationRunData/useEvalRunScenarioData.tsx
+++ b/web/ee/src/lib/hooks/useEvaluationRunData/useEvalRunScenarioData.tsx
@@ -7,11 +7,8 @@ import {UseEvaluationRunScenarioStepsFetcherResult} from "../useEvaluationRunSce
import {getCurrentRunId} from "./assets/atoms/migrationHelper"
import {scenarioStepFamily} from "./assets/atoms/runScopedScenarios"
-import {evalAtomStore} from "./assets/atoms/store"
const useEvalRunScenarioData = (scenarioId: string, runId?: string) => {
- const store = evalAtomStore()
-
// Memoize runId calculation to prevent infinite loops
const effectiveRunId = useMemo(() => {
if (runId) return runId
@@ -26,7 +23,6 @@ const useEvalRunScenarioData = (scenarioId: string, runId?: string) => {
// Read from the same global store that writes are going to
const stepLoadable = useAtomValue(
loadable(scenarioStepFamily({scenarioId, runId: effectiveRunId || ""})),
- {store},
)
return useMemo(() => {
diff --git a/web/ee/src/lib/hooks/useInvocationResult/index.ts b/web/ee/src/lib/hooks/useInvocationResult/index.ts
index 118137f9ea..55420f754e 100644
--- a/web/ee/src/lib/hooks/useInvocationResult/index.ts
+++ b/web/ee/src/lib/hooks/useInvocationResult/index.ts
@@ -9,7 +9,6 @@ import {readInvocationResponse} from "@/oss/lib/helpers/traceUtils"
import {getCurrentRunId} from "../useEvaluationRunData/assets/atoms/migrationHelper"
import {scenarioStatusAtomFamily} from "../useEvaluationRunData/assets/atoms/progress"
-import {evalAtomStore} from "../useEvaluationRunData/assets/atoms/store"
import useEvalRunScenarioData from "../useEvaluationRunData/useEvalRunScenarioData"
import type {UseInvocationResult, UseInvocationResultArgs} from "./types"
@@ -21,8 +20,6 @@ export function useInvocationResult({
editorType = "shared",
viewType = "single",
}: UseInvocationResultArgs): UseInvocationResult {
- const store = evalAtomStore()
-
// Use provided runId or fallback to current run context (memoized to prevent infinite loops)
const contextRunId = useRunId()
const runId = useMemo(() => {
@@ -45,7 +42,6 @@ export function useInvocationResult({
() => scenarioStatusAtomFamily({scenarioId, runId: runId || ""}),
[scenarioId, runId],
),
- {store},
) as any
// Early return if no runId is available
diff --git a/web/ee/src/lib/hooks/usePreviewEvaluations/assets/utils.ts b/web/ee/src/lib/hooks/usePreviewEvaluations/assets/utils.ts
index 3ffc624ec7..e50327161f 100644
--- a/web/ee/src/lib/hooks/usePreviewEvaluations/assets/utils.ts
+++ b/web/ee/src/lib/hooks/usePreviewEvaluations/assets/utils.ts
@@ -23,6 +23,96 @@ import {variantFlagsAtomFamily} from "@/oss/state/newPlayground/core/variantFlag
import {useOrgData} from "@/oss/state/org"
import {getProjectValues} from "@/oss/state/project"
+const pickString = (...values: unknown[]): string | undefined => {
+ for (const value of values) {
+ if (typeof value !== "string") continue
+ const trimmed = value.trim()
+ if (!trimmed) continue
+ return trimmed
+ }
+ return undefined
+}
+
+const deriveInvocationMetadata = (runIndex?: RunIndex) => {
+ if (!runIndex) return null
+ const [firstInvocationKey] = Array.from(runIndex.invocationKeys)
+ if (!firstInvocationKey) return null
+ const meta = runIndex.steps[firstInvocationKey]
+ if (!meta) return null
+ const refs = meta.refs ?? {}
+
+ const applicationRevision =
+ refs.applicationRevision || refs.application_revision || refs.revision
+ const applicationRef =
+ refs.application ||
+ applicationRevision?.application ||
+ refs.applicationRef ||
+ refs.application_ref
+ const variantRef =
+ refs.variant ||
+ refs.variantRef ||
+ refs.variant_ref ||
+ refs.applicationVariant ||
+ refs.application_variant
+
+ const appId = pickString(
+ applicationRef?.id,
+ applicationRef?.app_id,
+ applicationRef?.appId,
+ applicationRevision?.application_id,
+ applicationRevision?.applicationId,
+ refs.application?.id,
+ refs.application?.app_id,
+ refs.application?.appId,
+ )
+
+ const appName = pickString(
+ applicationRef?.name,
+ applicationRef?.slug,
+ refs.application?.name,
+ refs.application?.slug,
+ )
+
+ const variantName = pickString(
+ variantRef?.name,
+ variantRef?.slug,
+ variantRef?.variantName,
+ variantRef?.variant_name,
+ applicationRef?.name,
+ applicationRef?.slug,
+ refs.application?.name,
+ refs.application?.slug,
+ meta.key,
+ )
+
+ const revisionId = pickString(
+ variantRef?.id,
+ variantRef?.revisionId,
+ variantRef?.revision_id,
+ applicationRevision?.id,
+ applicationRevision?.revisionId,
+ applicationRevision?.revision_id,
+ )
+
+ const revisionLabel =
+ pickString(
+ variantRef?.version,
+ variantRef?.revision,
+ variantRef?.revisionLabel,
+ applicationRevision?.revision,
+ applicationRevision?.version,
+ applicationRevision?.name,
+ ) ?? null
+
+ return {
+ appId,
+ appName,
+ variantName,
+ revisionId,
+ revisionLabel: revisionLabel ?? undefined,
+ }
+}
+
export const enrichEvaluationRun = ({
run: _run,
testsets,
@@ -31,7 +121,7 @@ export const enrichEvaluationRun = ({
members,
runIndex,
extras,
- appNameById,
+ appNameById: _appNameById,
projectScope = false,
}: {
run: SnakeToCamelCaseKeys
@@ -49,6 +139,7 @@ export const enrichEvaluationRun = ({
projectScope?: boolean
}) => {
const run: Partial = _run
+ const appNameById = _appNameById ?? new Map()
// Convert snake_case keys to camelCase recursively
run.createdAtTimestamp = dayjs(run.createdAt, "YYYY/MM/DD H:mm:ssAZ").valueOf()
// Format creation date for display
@@ -163,6 +254,8 @@ export const enrichEvaluationRun = ({
const filteredVariants = combinedVariantList.filter((v) => uniqueRevisionIds.includes(v.id))
+ const invocationMetadata = deriveInvocationMetadata(runIndex)
+
const fallbackVariants =
filteredVariants.length || !runIndex
? []
@@ -208,18 +301,7 @@ export const enrichEvaluationRun = ({
const projectId = getProjectValues().projectId
const baseVariants = filteredVariants.length ? filteredVariants : []
- const combinedVariants = (baseVariants.length ? baseVariants : fallbackVariants).map(
- (variant) => {
- const mappedName = variant.appId ? appNameById?.get(variant.appId) : undefined
- if (mappedName && (!variant.appName || variant.appName === variant.appId)) {
- return {
- ...variant,
- appName: mappedName,
- }
- }
- return variant
- },
- ) as typeof fallbackVariants
+ const combinedVariants = (baseVariants.length ? baseVariants : fallbackVariants) as typeof fallbackVariants
const normalizedVariants = combinedVariants
.map((variant) => {
@@ -242,12 +324,76 @@ export const enrichEvaluationRun = ({
})
.filter((variant) => Boolean(variant.id))
- const primaryVariant = normalizedVariants[0]
+ const originalVariants = Array.isArray((run as any)?.variants)
+ ? ((run as any)?.variants as any[])
+ : []
+ const originalPrimaryVariant = originalVariants.length ? originalVariants[0] : undefined
+
+ const invocationAppId =
+ typeof invocationMetadata?.appId === "string" && invocationMetadata.appId
+ ? invocationMetadata.appId
+ : undefined
+ const invocationAppName =
+ invocationAppId && invocationMetadata?.appName ? invocationMetadata.appName : undefined
+
+ const runAppId =
+ typeof (run as any)?.appId === "string" && (run as any).appId.trim()
+ ? (run as any).appId.trim()
+ : undefined
+ const runAppName =
+ typeof (run as any)?.appName === "string" && (run as any).appName.trim()
+ ? (run as any).appName.trim()
+ : undefined
+
+ const originalVariantAppIds = originalVariants
+ .map(
+ (variant: any) =>
+ (typeof variant?.appId === "string" && variant.appId.trim()) ||
+ (typeof variant?.app_id === "string" && variant.app_id.trim()) ||
+ undefined,
+ )
+ .filter((value): value is string => Boolean(value))
+
+ const normalizedVariantAppIds = normalizedVariants
+ .map(
+ (variant: any) =>
+ (typeof variant?.appId === "string" && variant.appId.trim()) ||
+ (typeof variant?.applicationId === "string" && variant.applicationId.trim()) ||
+ undefined,
+ )
+ .filter((value): value is string => Boolean(value))
+
+ const finalAppId = pickString(
+ invocationAppId,
+ runAppId,
+ ...originalVariantAppIds,
+ ...normalizedVariantAppIds,
+ )
+
+ const variantForFinalApp =
+ finalAppId &&
+ (normalizedVariants.find((variant: any) => variant?.appId === finalAppId) ||
+ originalVariants.find(
+ (variant: any) =>
+ variant?.appId === finalAppId || variant?.app_id === finalAppId,
+ ))
+
+ const finalAppName = pickString(
+ invocationAppId && invocationAppId === finalAppId ? invocationAppName : undefined,
+ finalAppId ? appNameById.get(finalAppId) : undefined,
+ runAppId && runAppId === finalAppId ? runAppName : undefined,
+ variantForFinalApp?.appName,
+ variantForFinalApp?.appSlug,
+ originalPrimaryVariant?.variantName,
+ invocationMetadata?.variantName,
+ runAppName,
+ ((run as any)?.name as string) || undefined,
+ )
const returnValue = {
...run,
- appId: (run as any).appId || primaryVariant?.appId,
- appName: (run as any).appName || primaryVariant?.appName,
+ appId: finalAppId,
+ appName: finalAppName,
variants: normalizedVariants,
testsets: resolvedTestsets,
createdBy: members.find((member) => member.user.id === run.createdById),
@@ -265,11 +411,15 @@ export const enrichEvaluationRun = ({
if (!returnValue.appId && variant.appId) {
returnValue.appId = variant.appId
}
- if (!returnValue.appName && variant.appName) {
+ if (
+ !returnValue.appName &&
+ variant.appName &&
+ (!returnValue.appId || variant.appId === returnValue.appId)
+ ) {
returnValue.appName = variant.appName
}
})
- if (!returnValue.appName && returnValue.appId && appNameById) {
+ if (!returnValue.appName && returnValue.appId) {
const mappedName = appNameById.get(returnValue.appId)
if (mappedName) {
returnValue.appName = mappedName
diff --git a/web/ee/src/lib/hooks/usePreviewEvaluations/index.ts b/web/ee/src/lib/hooks/usePreviewEvaluations/index.ts
index 65438ca047..a0d0962cbd 100644
--- a/web/ee/src/lib/hooks/usePreviewEvaluations/index.ts
+++ b/web/ee/src/lib/hooks/usePreviewEvaluations/index.ts
@@ -10,7 +10,6 @@ import {useAppId} from "@/oss/hooks/useAppId"
import axios from "@/oss/lib/api/assets/axiosConfig"
import {EvaluationType} from "@/oss/lib/enums"
import {snakeToCamelCaseKeys} from "@/oss/lib/helpers/casing"
-import useEvaluators from "@/oss/lib/hooks/useEvaluators"
import {EvaluationStatus, SnakeToCamelCaseKeys, TestSet} from "@/oss/lib/Types"
import {slugify} from "@/oss/lib/utils/slugify"
import {createEvaluationRunConfig} from "@/oss/services/evaluationRuns/api"
@@ -154,13 +153,6 @@ const usePreviewEvaluations = ({
const routeAppId = useAppId()
const appId = (appIdOverride ?? routeAppId) || undefined
- const {data: humanEvaluators} = useEvaluators({
- preview: true,
- queries: {
- is_human: !types.includes(EvaluationType.automatic),
- },
- })
-
const referenceFilters = useMemo(() => {
const filters: any[] = []
if (appId) {
@@ -168,21 +160,8 @@ const usePreviewEvaluations = ({
application: {id: appId},
})
}
- if (types.includes(EvaluationType.human)) {
- if (Array.isArray(humanEvaluators) && humanEvaluators.length > 0) {
- humanEvaluators.forEach((ev) => {
- filters.push({
- evaluator: {id: ev.id},
- })
- })
- } else {
- filters.push({
- evaluator: {},
- })
- }
- }
return filters
- }, [appId, humanEvaluators, types])
+ }, [appId])
const typesKey = useMemo(() => types.slice().sort().join("|"), [types])
const queryEnabled = !skip && Boolean(projectId)
diff --git a/web/ee/src/state/url/focusDrawer.ts b/web/ee/src/state/url/focusDrawer.ts
index d847262900..397fa4b16c 100644
--- a/web/ee/src/state/url/focusDrawer.ts
+++ b/web/ee/src/state/url/focusDrawer.ts
@@ -131,11 +131,11 @@ export const clearFocusDrawerQueryParams = () => {
}
export type FocusDrawerPublicApi = {
- syncFocusDrawerStateFromUrl: (nextUrl?: string) => void;
- clearFocusDrawerQueryParams: () => void;
-};
+ syncFocusDrawerStateFromUrl: (nextUrl?: string) => void
+ clearFocusDrawerQueryParams: () => void
+}
export default {
syncFocusDrawerStateFromUrl,
clearFocusDrawerQueryParams,
-} satisfies FocusDrawerPublicApi;
+} satisfies FocusDrawerPublicApi
diff --git a/web/oss/package.json b/web/oss/package.json
index f5af13ff7e..ad24ead415 100644
--- a/web/oss/package.json
+++ b/web/oss/package.json
@@ -1,6 +1,6 @@
{
"name": "@agenta/oss",
- "version": "0.59.1",
+ "version": "0.59.2",
"private": true,
"engines": {
"node": ">=18"
diff --git a/web/oss/src/components/Playground/Components/PlaygroundVariantConfigPrompt/assets/PlaygroundVariantConfigPromptCollapseContent.tsx b/web/oss/src/components/Playground/Components/PlaygroundVariantConfigPrompt/assets/PlaygroundVariantConfigPromptCollapseContent.tsx
index 7b85e8a1c2..e41e190959 100644
--- a/web/oss/src/components/Playground/Components/PlaygroundVariantConfigPrompt/assets/PlaygroundVariantConfigPromptCollapseContent.tsx
+++ b/web/oss/src/components/Playground/Components/PlaygroundVariantConfigPrompt/assets/PlaygroundVariantConfigPromptCollapseContent.tsx
@@ -72,8 +72,8 @@ const PlaygroundVariantConfigPromptCollapseContent: React.FC
- Insert a {"{{variable}}"} in
- your template to create an input.
+ Insert a {"{{variable}}"} in your
+ template to create an input.
}
type="info"
diff --git a/web/oss/src/components/Playground/adapters/TurnMessageAdapter.tsx b/web/oss/src/components/Playground/adapters/TurnMessageAdapter.tsx
index 85580706d6..b769973c5f 100644
--- a/web/oss/src/components/Playground/adapters/TurnMessageAdapter.tsx
+++ b/web/oss/src/components/Playground/adapters/TurnMessageAdapter.tsx
@@ -305,9 +305,7 @@ const TurnMessageAdapter: React.FC = ({
const effectivePlaceholder = useMemo(() => {
if (placeholder) return placeholder
- return kind === "user"
- ? "Type your message…"
- : "Enter tool call result here and hit run"
+ return kind === "user" ? "Type your message…" : "Enter tool call result here and hit run"
}, [placeholder, kind])
// TODO: IMPROVE THIS
diff --git a/web/oss/src/components/pages/settings/WorkspaceManage/cellRenderers.tsx b/web/oss/src/components/pages/settings/WorkspaceManage/cellRenderers.tsx
index cd897096f2..ed8a00fbbd 100644
--- a/web/oss/src/components/pages/settings/WorkspaceManage/cellRenderers.tsx
+++ b/web/oss/src/components/pages/settings/WorkspaceManage/cellRenderers.tsx
@@ -55,8 +55,8 @@ export const Actions: React.FC<{
title: "Remove member",
message: `Are you sure you want to remove ${user.username} from this workspace?`,
onOk: () =>
- removeFromWorkspace({organizationId, workspaceId, email: user.email}, true).then(() =>
- refetch(),
+ removeFromWorkspace({organizationId, workspaceId, email: user.email}, true).then(
+ () => refetch(),
),
okText: "Remove",
})
@@ -127,7 +127,12 @@ export const Roles: React.FC<{
const handleChangeRole = async (roleName: string) => {
setLoading(true)
try {
- await assignWorkspaceRole({organizationId, workspaceId, email: user.email, role: roleName})
+ await assignWorkspaceRole({
+ organizationId,
+ workspaceId,
+ email: user.email,
+ role: roleName,
+ })
await Promise.all(
member.roles
.filter((item) => item.role_name !== roleName)
diff --git a/web/oss/src/lib/helpers/useSubscriptionDataWrapper.ts b/web/oss/src/lib/helpers/useSubscriptionDataWrapper.ts
index 9e5a57458d..f334db6ed1 100644
--- a/web/oss/src/lib/helpers/useSubscriptionDataWrapper.ts
+++ b/web/oss/src/lib/helpers/useSubscriptionDataWrapper.ts
@@ -1,9 +1,24 @@
+import {isDemo} from "./utils"
+
+const DEFAULT_SUBSCRIPTION_STATE = {
+ subscription: undefined,
+ isSubLoading: false,
+ mutateSubscription: () => undefined,
+ error: undefined,
+ isError: false,
+ isSuccess: false,
+} as const
+
export function useSubscriptionDataWrapper() {
+ if (!isDemo()) {
+ return DEFAULT_SUBSCRIPTION_STATE
+ }
+
try {
// eslint-disable-next-line @typescript-eslint/no-require-imports
const mod = require("@/agenta-oss-common/services/billing")
return mod.useSubscriptionData()
} catch {
- return {subscription: undefined}
+ return DEFAULT_SUBSCRIPTION_STATE
}
}
diff --git a/web/oss/src/pages/auth/[[...path]].tsx b/web/oss/src/pages/auth/[[...path]].tsx
index f7da2782d8..ef1ed9af39 100644
--- a/web/oss/src/pages/auth/[[...path]].tsx
+++ b/web/oss/src/pages/auth/[[...path]].tsx
@@ -55,7 +55,16 @@ const Auth = () => {
email: emailFromQuery,
})
}
- }, [isInvitedUser, invite, setInvite, token, organizationId, projectId, workspaceId, emailFromQuery])
+ }, [
+ isInvitedUser,
+ invite,
+ setInvite,
+ token,
+ organizationId,
+ projectId,
+ workspaceId,
+ emailFromQuery,
+ ])
const authErrorMsg = (error: any) => {
if (error.isSuperTokensGeneralError === true) {
diff --git a/web/oss/src/services/api.ts b/web/oss/src/services/api.ts
index 5d9a39d7d1..1e38b89a1e 100644
--- a/web/oss/src/services/api.ts
+++ b/web/oss/src/services/api.ts
@@ -361,6 +361,9 @@ export const fetchAppContainerURL = async (
// Retrieve container URL from backend
const {data} = await axios.get(
`${getAgentaApiUrl()}/variants/${variantId}?project_id=${projectId}`,
+ {
+ _ignoreError: true,
+ } as any,
)
const uriObject = await findCustomWorkflowPath(data.uri)
if (uriObject) {
diff --git a/web/oss/src/state/url/auth.ts b/web/oss/src/state/url/auth.ts
index b5a2655cdf..184a1e0c8e 100644
--- a/web/oss/src/state/url/auth.ts
+++ b/web/oss/src/state/url/auth.ts
@@ -107,10 +107,10 @@ export const syncAuthStateFromUrl = (nextUrl?: string) => {
const user = store.get(userAtom)
const urlState = store.get(urlAtom)
- const resolvedPath = nextUrl ? url.pathname : appState.pathname ?? url.pathname
+ const resolvedPath = nextUrl ? url.pathname : (appState.pathname ?? url.pathname)
const resolvedAsPath = nextUrl
? `${url.pathname}${url.search}${url.hash}`
- : appState.asPath ?? `${url.pathname}${url.search}${url.hash}`
+ : (appState.asPath ?? `${url.pathname}${url.search}${url.hash}`)
const path = resolvedPath
const asPath = resolvedAsPath
const isAuthRoute = path.startsWith("/auth")
diff --git a/web/package.json b/web/package.json
index 1c1974c6d1..af0c25fc88 100644
--- a/web/package.json
+++ b/web/package.json
@@ -1,6 +1,6 @@
{
"name": "agenta-web",
- "version": "0.59.1",
+ "version": "0.59.2",
"workspaces": [
"ee",
"oss",