Improve eval performance and debugging

Author: gspencergoog

specification/0.9/eval/src/flows.ts

@@ -62,52 +62,20 @@ export const componentGeneratorFlow = ai.defineFlow(
       .map((s: any) => JSON.stringify(s, null, 2))
       .join("\n\n");
 
-    const fullPrompt = `You are an AI assistant. Based on the following request, generate a JSON object that conforms to the provided JSON Schemas. The output MUST be ONLY the JSON object enclosed in a markdown code block.
-
-DO NOT include any other text before or after the markdown code block.
-
-Example Output:
-\`\`\`json
-{
-  "updateComponents": {
-    "surfaceId": "contact_form_1",
-    "components": [
-      {
-        "id": "root",
-        "props": {
-          "component": "Column",
-          "children": [
-            "first_name_label",
-            "first_name_field"
-          ]
-        }
-      },
-      {
-        "id": "first_name_label",
-        "props": {
-          "component": "Text",
-          "text": "First Name"
-        }
-      },
-      {
-        "id": "first_name_field",
-        "props": {
-          "component": "TextField",
-          "label": "First Name",
-          "text": { "path": "/contact/firstName" },
-          "textFieldType": "shortText"
-        }
-      }
-    ]
-  }
-}
-\`\`\`
+    const fullPrompt = `You are an AI assistant. Based on the following request, generate a stream of JSON messages that conform to the provided JSON Schemas.
+The output MUST be a series of JSON objects, each enclosed in a markdown code block (or a single block with multiple objects).
 
-Request:
-${prompt}
+Standard Instructions:
+1. Always start by generating a 'createSurface' message with surfaceId 'main'.
+2. Then, generate a 'updateComponents' message with surfaceId 'main' containing the requested UI.
+3. Ensure all component children are referenced by ID (using the 'children' or 'child' property with IDs), NOT nested inline as objects.
+4. If the request involves data binding, you may also generate 'updateDataModel' messages.
 
-JSON Schemas:
+Schemas:
 ${schemaDefs}
+
+Request:
+${prompt}
 `;
     const estimatedInputTokens = Math.ceil(fullPrompt.length / 2.5);
     await rateLimiter.acquirePermit(

specification/0.9/eval/src/message_type_matcher.ts

@@ -20,7 +20,7 @@ import { SchemaMatcher, ValidationResult } from "./schema_matcher";
  * A concrete matcher that verifies the top-level message type.
  */
 export class MessageTypeMatcher extends SchemaMatcher {
-  constructor(private messageType: string) {
+  constructor(public messageType: string) {
     super();
   }
 
@@ -45,6 +45,6 @@ export class MessageTypeMatcher extends SchemaMatcher {
   }
 
   get description(): string {
-    return `Expected top-level message type to be '${this.messageType}'`;
+    return `Message type '${this.messageType}'`;
   }
 }

specification/0.9/eval/src/prompts.ts

@@ -20,5 +20,14 @@ export interface ValidationResult {
 }
 
 export abstract class SchemaMatcher {
+  /**
+   * A human-readable description of what this matcher is looking for.
+   */
+  abstract get description(): string;
+
+  /**
+   * Validates that the given schema satisfies the matcher's criteria.
+   * @param schema The schema to validate (e.g. a message object).
+   */
   abstract validate(schema: any): ValidationResult;
 }

specification/0.9/eval/src/schema_matcher.ts

@@ -31,6 +31,21 @@ export class SurfaceUpdateSchemaMatcher extends SchemaMatcher {
     super();
   }
 
+  get description(): string {
+    let desc = `Component '${this.componentType}'`;
+    if (this.propertyName) {
+      desc += ` with property '${this.propertyName}'`;
+      if (this.propertyValue !== undefined) {
+        if (typeof this.propertyValue === "function") {
+          desc += ` matching predicate`;
+        } else {
+          desc += ` matching ${JSON.stringify(this.propertyValue)}`;
+        }
+      }
+    }
+    return desc;
+  }
+
   private getComponentById(components: any[], id: string): any | undefined {
     return components.find((c: any) => c.id === id);
   }
@@ -128,7 +143,12 @@ export class SurfaceUpdateSchemaMatcher extends SchemaMatcher {
         : s1 === s2;
     };
 
-    // 1. Direct Primitive Match (Shorthand)
+    // Predicate Function Match
+    if (typeof expectedValue === "function") {
+      return expectedValue(actualValue);
+    }
+
+    // Direct Primitive Match (Shorthand)
     if (typeof actualValue === "string" && typeof expectedValue === "string") {
       return compareStrings(actualValue, expectedValue);
     }
@@ -142,7 +162,7 @@ export class SurfaceUpdateSchemaMatcher extends SchemaMatcher {
       return actualValue === expectedValue;
     }
 
-    // 2. Object with Path (Should not match a literal expected value usually, unless expectedValue is the path object)
+    // Object with Path (Should not match a literal expected value usually, unless expectedValue is the path object)
     if (
       typeof actualValue === "object" &&
       !Array.isArray(actualValue) &&
@@ -152,7 +172,7 @@ export class SurfaceUpdateSchemaMatcher extends SchemaMatcher {
       return false;
     }
 
-    // 3. Array Match (e.g. MultipleChoice options)
+    // Array Match (e.g. MultipleChoice options)
     if (Array.isArray(actualValue)) {
       for (const item of actualValue) {
         // Direct match in array

specification/0.9/eval/src/surface_update_schema_matcher.ts

@@ -20,7 +20,7 @@ import path from "path";
 import { SurfaceUpdateSchemaMatcher } from "./surface_update_schema_matcher";
 import { SchemaMatcher } from "./schema_matcher";
 
-const ajv = new Ajv({ strict: false });
+const ajv = new Ajv({ strict: false, verbose: true });
 
 const schemaDir = path.resolve(process.cwd(), "../");
 const serverToClientSchema = JSON.parse(
@@ -58,7 +58,16 @@ export function validateSchema(
     if (!valid) {
       if (validate.errors) {
         validate.errors.forEach((err) => {
-          errors.push(`AJV: ${err.instancePath} ${err.message}`);
+          let errorMsg = `AJV [Message ${messages.indexOf(message)}]: ${
+            err.instancePath
+          } ${err.message}`;
+          if (err.params) {
+            errorMsg += ` | Params: ${JSON.stringify(err.params)}`;
+          }
+          if (err.data !== undefined) {
+            errorMsg += ` | Data: ${JSON.stringify(err.data)}`;
+          }
+          errors.push(errorMsg);
         });
       }
     }
@@ -97,7 +106,7 @@ export function validateSchema(
         // Actually, let's just say "Matcher failed".
         // If we really want, we could change SchemaMatcher to have a description.
         // But for now:
-        errors.push(`Matcher failed to match any message in the stream.`);
+        errors.push(`Matcher failed: ${matcher.description}`);
       }
     }
   }
@@ -239,7 +248,7 @@ function validateComponent(
     for (const prop of props) {
       if (properties[prop] === undefined) {
         errors.push(
-          `Component '${id}' of type '${componentType}' is missing required property '${prop}'.`
+          `Component ${JSON.stringify(id)} of type '${componentType}' is missing required property '${prop}'.`
         );
       }
     }
@@ -249,7 +258,7 @@ function validateComponent(
     for (const id of ids) {
       if (id && !allIds.has(id)) {
         errors.push(
-          `Component '${id}' references non-existent component ID '${id}'.`
+          `Component ${JSON.stringify(id)} references non-existent component ID.`
         );
       }
     }

specification/0.9/eval/src/validator.ts

@@ -199,14 +199,15 @@
         },
         "distribution": {
           "type": "string",
-          "description": "Defines the arrangement of children along the main axis (horizontally). This corresponds to the CSS 'justify-content' property.",
+          "description": "Defines the arrangement of children along the main axis (horizontally). This corresponds to the CSS 'justify-content' property, but uses camelCase values (e.g., 'spaceBetween').",
           "enum": [
             "center",
             "end",
             "spaceAround",
             "spaceBetween",
             "spaceEvenly",
-            "start"
+            "start",
+            "stretch"
           ]
         },
         "alignment": {
@@ -228,14 +229,15 @@
         },
         "distribution": {
           "type": "string",
-          "description": "Defines the arrangement of children along the main axis (vertically). This corresponds to the CSS 'justify-content' property.",
+          "description": "Defines the arrangement of children along the main axis (vertically). This corresponds to the CSS 'justify-content' property, but uses camelCase values (e.g., 'spaceBetween').",
           "enum": [
             "start",
             "center",
             "end",
             "spaceBetween",
             "spaceAround",
-            "spaceEvenly"
+            "spaceEvenly",
+            "stretch"
           ]
         },
         "alignment": {
@@ -433,7 +435,7 @@
         "component": { "const": "DateTimeInput" },
         "value": {
           "$ref": "common_types.json#/$defs/stringOrPath",
-          "description": "The selected date and/or time value."
+          "description": "The selected date and/or time value. If not yet set, initialize with an empty string."
         },
         "enableDate": {
           "type": "boolean",