Skip to content

Simplify default composite evaluation management #2093

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
andresgutgon merged 1 commit into from
Jan 8, 2026
Merged

Simplify default composite evaluation management #2093

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion ...ay/src/routes/api/v3/projects/versions/documents/createOrUpdate/createOrUpdate.handler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,9 @@ export const createOrUpdateDocumentHandler: AppRouteHandler<
document = await updateDocumentUnsafe({
commit,
document: existingDoc,
content: prompt,
data: {
content: prompt,
},
}).then((r) => r.unwrap())
} else {
// Document doesn't exist, create it using unsafe version
Expand Down
63 changes: 2 additions & 61 deletions ...nts/[documentUuid]/(withTabs)/evaluations/[evaluationUuid]/_components/EvaluationPage.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import {
} from '$/components/Providers/WebsocketsProvider/useSockets'
import { RealtimeToggle } from '$/components/RealtimeToggle'
import { useEvaluationResultsV2 } from '$/stores/evaluationResultsV2'
import { useEvaluationsV2, useEvaluationV2Stats } from '$/stores/evaluationsV2'
import { useEvaluationV2Stats } from '$/stores/evaluationsV2'
import {
EvaluationMetric,
EvaluationType,
Expand All @@ -21,12 +21,10 @@ import {
evaluationResultsV2SearchToQueryParams,
} from '@latitude-data/core/helpers'
import { EvaluationResultV2WithDetails } from '@latitude-data/core/schema/types'
import { Alert } from '@latitude-data/web-ui/atoms/Alert'
import { Button } from '@latitude-data/web-ui/atoms/Button'
import { Icon } from '@latitude-data/web-ui/atoms/Icons'
import { Text } from '@latitude-data/web-ui/atoms/Text'
import { TableWithHeader } from '@latitude-data/web-ui/molecules/ListingHeader'
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
import { useCallback, useEffect, useRef, useState } from 'react'
import { useDebounce, useDebouncedCallback } from 'use-debounce'
import { EvaluationTitle } from '../../_components/EvaluationTitle'
import { EvaluationActions } from './EvaluationActions'
Expand Down Expand Up @@ -194,65 +192,8 @@ export function EvaluationPage<
})

const openSettingsRef = useRef<() => void>(undefined)
const { data: evaluations, isLoading: isLoadingEvaluations } =
useEvaluationsV2({ project, commit, document })
const isTargetSynced = useMemo(() => {
if (isLoadingEvaluations) return true
if (evaluation.type !== EvaluationType.Composite) return true
if (
!(evaluation as EvaluationV2<EvaluationType.Composite>).configuration
.defaultTarget
) {
return true
}

const linkedUuids = evaluations
.filter((e) => e.uuid !== evaluation.uuid && !e.deletedAt && e.issueId)
.map((e) => e.uuid)

const linkedSet = new Set(linkedUuids)
const currentSet = new Set(
(
evaluation as EvaluationV2<EvaluationType.Composite>
).configuration.evaluationUuids,
)

if (linkedSet.size !== currentSet.size) return false

for (const uuid of linkedSet) {
if (!currentSet.has(uuid)) return false
}

return true
}, [isLoadingEvaluations, evaluation, evaluations])

return (
<div className='flex flex-grow min-h-0 flex-col w-full gap-4 p-6'>
{evaluation.type === EvaluationType.Composite && !isTargetSynced && (
<Alert
spacing='xsmall'
variant='warning'
direction='column'
description='This composite score is unsynced from evaluations that are tracking and monitoring active issues'
cta={
<Button
variant='link'
size='none'
textColor='warningMutedForeground'
iconProps={{
name: 'arrowRight',
color: 'warningMutedForeground',
size: 'normal',
placement: 'right',
strokeWidth: 2.5,
}}
onClick={() => openSettingsRef.current?.()}
>
Sync score
</Button>
}
/>
)}
<TableWithHeader
verticalAligment='bottom'
title={<EvaluationTitle evaluation={evaluation} />}
Expand Down
162 changes: 10 additions & 152 deletions ...mitUuid]/documents/[documentUuid]/(withTabs)/evaluations/_components/EvaluationsTable.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import { useNavigate } from '$/hooks/useNavigate'
import { ROUTES } from '$/services/routes'
import { useEvaluationsV2 } from '$/stores/evaluationsV2'
import { EvaluationType, EvaluationV2 } from '@latitude-data/core/constants'
import { Alert } from '@latitude-data/web-ui/atoms/Alert'
import { Badge } from '@latitude-data/web-ui/atoms/Badge'
import { Button } from '@latitude-data/web-ui/atoms/Button'
import {
Expand All @@ -33,7 +32,6 @@ import {
BlankSlateStep,
BlankSlateWithSteps,
} from '@latitude-data/web-ui/molecules/BlankSlateWithSteps'
import Link from 'next/link'
import { useCallback, useMemo, useState } from 'react'
import { EvaluationsGenerator } from './EvaluationsGenerator'

Expand Down Expand Up @@ -93,38 +91,6 @@ export function EvaluationsTable({
.enableControls,
) as EvaluationV2<EvaluationType.Human> | undefined
}, [evaluations])
const optimizationsEvaluation = useMemo(() => {
return evaluations.find(
(e) =>
e.type === EvaluationType.Composite &&
!!(e as EvaluationV2<EvaluationType.Composite>).configuration
.defaultTarget,
) as EvaluationV2<EvaluationType.Composite> | undefined
}, [evaluations])

const isTargetSynced = useMemo(() => {
if (!optimizationsEvaluation) return true

const linkedUuids = evaluations
.filter(
(e) =>
e.uuid !== optimizationsEvaluation.uuid && !e.deletedAt && e.issueId,
)
.map((e) => e.uuid)

const linkedSet = new Set(linkedUuids)
const currentSet = new Set(
optimizationsEvaluation.configuration.evaluationUuids,
)

if (linkedSet.size !== currentSet.size) return false

for (const uuid of linkedSet) {
if (!currentSet.has(uuid)) return false
}

return true
}, [evaluations, optimizationsEvaluation])

const [openDeleteModal, setOpenDeleteModal] = useState(false)
const onDelete = useCallback(
Expand Down Expand Up @@ -184,45 +150,6 @@ export function EvaluationsTable({
],
)

const onUnuseOptimizations = useCallback(
async (evaluation: EvaluationV2<EvaluationType.Composite>) => {
if (evaluation.uuid !== optimizationsEvaluation?.uuid) return
if (isUpdatingEvaluation) return
return await updateEvaluation({
documentUuid: document.documentUuid,
evaluationUuid: evaluation.uuid,
settings: {
configuration: { ...evaluation.configuration, defaultTarget: false },
},
})
},
[document, isUpdatingEvaluation, updateEvaluation, optimizationsEvaluation],
)

const onUseOptimizations = useCallback(
async (evaluation: EvaluationV2<EvaluationType.Composite>) => {
if (evaluation.uuid === optimizationsEvaluation?.uuid) return
if (isUpdatingEvaluation) return
if (optimizationsEvaluation) {
await onUnuseOptimizations(optimizationsEvaluation)
}
return await updateEvaluation({
documentUuid: document.documentUuid,
evaluationUuid: evaluation.uuid,
settings: {
configuration: { ...evaluation.configuration, defaultTarget: true },
},
})
},
[
document,
isUpdatingEvaluation,
updateEvaluation,
optimizationsEvaluation,
onUnuseOptimizations,
],
)

const onToggleLiveEvaluation = useCallback(
async (evaluation: EvaluationV2) => {
if (isUpdatingEvaluation) return
Expand Down Expand Up @@ -267,47 +194,6 @@ export function EvaluationsTable({
/>
<Text.H5M>{EVALUATION_SPECIFICATIONS[type].name}</Text.H5M>
</span>
{type === EvaluationType.Composite &&
optimizationsEvaluation &&
!isTargetSynced && (
<Alert
spacing='xsmall'
variant='warning'
direction='column'
description={`${optimizationsEvaluation.name} score is unsynced from evaluations that are tracking and monitoring active issues`}
cta={
<Link
href={
ROUTES.projects
.detail({ id: project.id })
.commits.detail({ uuid: commit.uuid })
.documents.detail({
uuid: document.documentUuid,
})
.evaluations.detail({
uuid: optimizationsEvaluation.uuid,
}).root + '?action=editSettings'
}
className='leading-none'
>
<Button
variant='link'
size='none'
textColor='warningMutedForeground'
iconProps={{
name: 'arrowRight',
color: 'warningMutedForeground',
size: 'normal',
placement: 'right',
strokeWidth: 2.5,
}}
>
Sync score
</Button>
</Link>
}
/>
)}
<Table className='table-fixed'>
<TableHeader className='isolate sticky top-0 z-10'>
<TableRow>
Expand Down Expand Up @@ -371,33 +257,32 @@ export function EvaluationsTable({
This evaluation is used for annotations
</Tooltip>
)}
{evaluation.uuid ===
optimizationsEvaluation?.uuid && (
{!!evaluation.evaluateLiveLogs && (
<Tooltip
asChild
trigger={
<Badge variant='accent'>
Optimizations
</Badge>
<Badge variant='accent'>Live</Badge>
}
maxWidth='none'
align='center'
side='top'
>
This evaluation is the default for
optimizations and distillations
This evaluation is running on live logs
</Tooltip>
)}
{!!evaluation.evaluateLiveLogs && (
{evaluation.uuid ===
document.mainEvaluationUuid && (
<Tooltip
asChild
trigger={
<Badge variant='accent'>Live</Badge>
<Badge variant='accent'>
Optimizations
</Badge>
}
align='center'
side='top'
>
This evaluation is running on live logs
This evaluation is the default for
optimizations and distillations
</Tooltip>
)}
</div>
Expand Down Expand Up @@ -449,33 +334,6 @@ export function EvaluationsTable({
},
] as MenuOption[])
: []),
...(evaluation.type ===
EvaluationType.Composite
? ([
evaluation.uuid ===
optimizationsEvaluation?.uuid
? {
label: 'Unuse for optimizations',
onElementClick: (e) =>
e.stopPropagation(),
onClick: () =>
onUnuseOptimizations(
evaluation as EvaluationV2<EvaluationType.Composite>,
),
disabled: isUpdatingEvaluation,
}
: {
label: 'Use for optimizations',
onElementClick: (e) =>
e.stopPropagation(),
onClick: () =>
onUseOptimizations(
evaluation as EvaluationV2<EvaluationType.Composite>,
),
disabled: isUpdatingEvaluation,
},
] as MenuOption[])
: []),
...(specification.supportsLiveEvaluation
? ([
{
Expand Down
12 changes: 4 additions & 8 deletions ...ocumentUuid]/(withTabs)/optimizations/_components/OptimizationForm/EvaluationSelector.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import { getEvaluationMetricSpecification } from '$/components/evaluations'
import { useEvaluationsV2 } from '$/stores/evaluationsV2'
import { EvaluationType, EvaluationV2 } from '@latitude-data/core/constants'
import { Commit } from '@latitude-data/core/schema/models/types/Commit'
import { DocumentVersion } from '@latitude-data/core/schema/models/types/DocumentVersion'
import { Project } from '@latitude-data/core/schema/models/types/Project'
Expand Down Expand Up @@ -33,13 +32,10 @@ export function EvaluationSelector({
})

const defaultEvaluation = useMemo(() => {
return evaluations.find(
(e) =>
e.type === EvaluationType.Composite &&
!!(e as EvaluationV2<EvaluationType.Composite>).configuration
.defaultTarget,
) as EvaluationV2<EvaluationType.Composite> | undefined
}, [evaluations])
if (!document.mainEvaluationUuid) return undefined

return evaluations.find((e) => e.uuid === document.mainEvaluationUuid)
}, [evaluations, document.mainEvaluationUuid])

useEffect(() => {
if (value) return
Expand Down
Loading
Loading