Improve evals settings import (#2298)

Author: cte

evals/apps/web/src/app/runs/new/new-run.tsx

@@ -9,7 +9,7 @@ import fuzzysort from "fuzzysort"
 import { toast } from "sonner"
 import { X, Rocket, Check, ChevronsUpDown, HardDriveUpload, CircleCheck } from "lucide-react"
 
-import { globalSettingsSchema, rooCodeDefaults } from "@evals/types"
+import { globalSettingsSchema, providerSettingsSchema, rooCodeDefaults } from "@evals/types"
 
 import { createRun } from "@/lib/server/runs"
 import { createRunSchema as formSchema, type CreateRun as FormValues } from "@/lib/schemas"
@@ -73,7 +73,6 @@ export function NewRun() {
 
 	const {
 		setValue,
-		setError,
 		clearErrors,
 		watch,
 		formState: { isSubmitting },
@@ -147,14 +146,51 @@ export function NewRun() {
 			clearErrors("settings")
 
 			try {
-				const result = z.object({ globalSettings: globalSettingsSchema }).parse(JSON.parse(await file.text()))
-				setValue("settings", result.globalSettings)
+				const { providerProfiles, globalSettings } = z
+					.object({
+						providerProfiles: z.object({
+							currentApiConfigName: z.string(),
+							apiConfigs: z.record(z.string(), providerSettingsSchema),
+						}),
+						globalSettings: globalSettingsSchema,
+					})
+					.parse(JSON.parse(await file.text()))
+
+				const providerSettings = providerProfiles.apiConfigs[providerProfiles.currentApiConfigName] ?? {}
+
+				if (providerSettings.apiProvider === "openrouter" && providerSettings.openRouterModelId) {
+					const {
+						openRouterModelId,
+						modelMaxTokens,
+						modelMaxThinkingTokens,
+						modelTemperature,
+						includeMaxTokens,
+					} = providerSettings
+
+					const model = openRouterModelId
+
+					const settings = {
+						...rooCodeDefaults,
+						openRouterModelId,
+						modelMaxTokens,
+						modelMaxThinkingTokens,
+						modelTemperature,
+						includeMaxTokens,
+						...globalSettings,
+					}
+
+					setValue("model", model)
+					setValue("settings", settings)
+				} else {
+					setValue("settings", globalSettings)
+				}
+
 				event.target.value = ""
-			} catch (_error) {
-				setError("settings", { message: "Error parsing JSON file. Please check the file format." })
+			} catch (e) {
+				toast.error(e instanceof Error ? e.message : "An unknown error occurred.")
 			}
 		},
-		[clearErrors, setError, setValue],
+		[clearErrors, setValue],
 	)
 
 	return (

evals/apps/web/src/app/runs/new/settings-diff.tsx

@@ -1,6 +1,6 @@
 import { Fragment, HTMLAttributes } from "react"
 
-import { RooCodeSettings } from "@evals/types"
+import { RooCodeSettings, ROO_CODE_SETTINGS_KEYS } from "@evals/types"
 
 import { cn } from "@/lib/utils"
 
@@ -23,7 +23,8 @@ export function SettingsDiff({
 			<div className="font-medium text-muted-foreground">Setting</div>
 			<div className="font-medium text-muted-foreground">Default</div>
 			<div className="font-medium text-muted-foreground">Custom</div>
-			{Object.entries(defaults).flatMap(([key, defaultValue]) => {
+			{ROO_CODE_SETTINGS_KEYS.map((key) => {
+				const defaultValue = defaults[key as keyof typeof defaults]
 				const customValue = custom[key as keyof typeof custom]
 				const isDefault = JSON.stringify(defaultValue) === JSON.stringify(customValue)
 
@@ -49,9 +50,15 @@ type SettingDiffProps = HTMLAttributes<HTMLDivElement> & {
 export function SettingDiff({ name, defaultValue, customValue, ...props }: SettingDiffProps) {
 	return (
 		<Fragment {...props}>
-			<div className="overflow-hidden font-mono">{name}</div>
-			<pre className="inline text-rose-500 line-through">{defaultValue}</pre>
-			<pre className="inline text-teal-500">{customValue}</pre>
+			<div className="overflow-hidden font-mono" title={name}>
+				{name}
+			</div>
+			<pre className="overflow-hidden inline text-rose-500 line-through" title={defaultValue}>
+				{defaultValue}
+			</pre>
+			<pre className="overflow-hidden inline text-teal-500" title={customValue}>
+				{customValue}
+			</pre>
 		</Fragment>
 	)
 }

evals/packages/types/src/roo-code.ts

@@ -373,11 +373,10 @@ export const providerSettingsSchema = z.object({
 	requestyApiKey: z.string().optional(),
 	requestyModelId: z.string().optional(),
 	requestyModelInfo: modelInfoSchema.optional(),
-	// Claude 3.7 Sonnet Thinking
-	modelTemperature: z.number().nullish(),
-	modelMaxTokens: z.number().optional(),
-	modelMaxThinkingTokens: z.number().optional(),
 	// Generic
+	modelMaxTokens: z.number().optional(), // Currently only used by Anthropic hybrid thinking models.
+	modelMaxThinkingTokens: z.number().optional(), // Currently only used by Anthropic hybrid thinking models.
+	modelTemperature: z.number().nullish(),
 	includeMaxTokens: z.boolean().optional(),
 	// Fake AI
 	fakeAi: z.unknown().optional(),
@@ -619,6 +618,8 @@ export const rooCodeSettingsSchema = providerSettingsSchema.merge(globalSettings
 
 export type RooCodeSettings = GlobalSettings & ProviderSettings
 
+export const ROO_CODE_SETTINGS_KEYS = [...GLOBAL_SETTINGS_KEYS, ...PROVIDER_SETTINGS_KEYS] as Keys<RooCodeSettings>[]
+
 /**
  * SecretState
  */