Skip to content

Improve evals settings import #2298

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cte merged 1 commit into from
Apr 4, 2025
Merged

Improve evals settings import #2298

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 43 additions & 7 deletions evals/apps/web/src/app/runs/new/new-run.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import fuzzysort from "fuzzysort"
import { toast } from "sonner"
import { X, Rocket, Check, ChevronsUpDown, HardDriveUpload, CircleCheck } from "lucide-react"

import { globalSettingsSchema, rooCodeDefaults } from "@evals/types"
import { globalSettingsSchema, providerSettingsSchema, rooCodeDefaults } from "@evals/types"

import { createRun } from "@/lib/server/runs"
import { createRunSchema as formSchema, type CreateRun as FormValues } from "@/lib/schemas"
Expand Down Expand Up @@ -73,7 +73,6 @@ export function NewRun() {

const {
setValue,
setError,
clearErrors,
watch,
formState: { isSubmitting },
Expand Down Expand Up @@ -147,14 +146,51 @@ export function NewRun() {
clearErrors("settings")

try {
const result = z.object({ globalSettings: globalSettingsSchema }).parse(JSON.parse(await file.text()))
setValue("settings", result.globalSettings)
const { providerProfiles, globalSettings } = z
.object({
providerProfiles: z.object({
currentApiConfigName: z.string(),
apiConfigs: z.record(z.string(), providerSettingsSchema),
}),
globalSettings: globalSettingsSchema,
})
.parse(JSON.parse(await file.text()))

const providerSettings = providerProfiles.apiConfigs[providerProfiles.currentApiConfigName] ?? {}

if (providerSettings.apiProvider === "openrouter" && providerSettings.openRouterModelId) {
const {
openRouterModelId,
modelMaxTokens,
modelMaxThinkingTokens,
modelTemperature,
includeMaxTokens,
} = providerSettings

const model = openRouterModelId

const settings = {
...rooCodeDefaults,
openRouterModelId,
modelMaxTokens,
modelMaxThinkingTokens,
modelTemperature,
includeMaxTokens,
...globalSettings,
}

setValue("model", model)
setValue("settings", settings)
} else {
setValue("settings", globalSettings)
}

event.target.value = ""
} catch (_error) {
setError("settings", { message: "Error parsing JSON file. Please check the file format." })
} catch (e) {
toast.error(e instanceof Error ? e.message : "An unknown error occurred.")
}
},
[clearErrors, setError, setValue],
[clearErrors, setValue],
)

return (
Expand Down
17 changes: 12 additions & 5 deletions evals/apps/web/src/app/runs/new/settings-diff.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { Fragment, HTMLAttributes } from "react"

import { RooCodeSettings } from "@evals/types"
import { RooCodeSettings, ROO_CODE_SETTINGS_KEYS } from "@evals/types"

import { cn } from "@/lib/utils"

Expand All @@ -23,7 +23,8 @@ export function SettingsDiff({
<div className="font-medium text-muted-foreground">Setting</div>
<div className="font-medium text-muted-foreground">Default</div>
<div className="font-medium text-muted-foreground">Custom</div>
{Object.entries(defaults).flatMap(([key, defaultValue]) => {
{ROO_CODE_SETTINGS_KEYS.map((key) => {
const defaultValue = defaults[key as keyof typeof defaults]
const customValue = custom[key as keyof typeof custom]
const isDefault = JSON.stringify(defaultValue) === JSON.stringify(customValue)

Expand All @@ -49,9 +50,15 @@ type SettingDiffProps = HTMLAttributes<HTMLDivElement> & {
export function SettingDiff({ name, defaultValue, customValue, ...props }: SettingDiffProps) {
return (
<Fragment {...props}>
<div className="overflow-hidden font-mono">{name}</div>
<pre className="inline text-rose-500 line-through">{defaultValue}</pre>
<pre className="inline text-teal-500">{customValue}</pre>
<div className="overflow-hidden font-mono" title={name}>
{name}
</div>
<pre className="overflow-hidden inline text-rose-500 line-through" title={defaultValue}>
{defaultValue}
</pre>
<pre className="overflow-hidden inline text-teal-500" title={customValue}>
{customValue}
</pre>
</Fragment>
)
}
9 changes: 5 additions & 4 deletions evals/packages/types/src/roo-code.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -373,11 +373,10 @@ export const providerSettingsSchema = z.object({
requestyApiKey: z.string().optional(),
requestyModelId: z.string().optional(),
requestyModelInfo: modelInfoSchema.optional(),
// Claude 3.7 Sonnet Thinking
modelTemperature: z.number().nullish(),
modelMaxTokens: z.number().optional(),
modelMaxThinkingTokens: z.number().optional(),
// Generic
modelMaxTokens: z.number().optional(), // Currently only used by Anthropic hybrid thinking models.
modelMaxThinkingTokens: z.number().optional(), // Currently only used by Anthropic hybrid thinking models.
modelTemperature: z.number().nullish(),
includeMaxTokens: z.boolean().optional(),
// Fake AI
fakeAi: z.unknown().optional(),
Expand Down Expand Up @@ -619,6 +618,8 @@ export const rooCodeSettingsSchema = providerSettingsSchema.merge(globalSettings

export type RooCodeSettings = GlobalSettings & ProviderSettings

export const ROO_CODE_SETTINGS_KEYS = [...GLOBAL_SETTINGS_KEYS, ...PROVIDER_SETTINGS_KEYS] as Keys<RooCodeSettings>[]

/**
* SecretState
*/
Expand Down