Add gulf genkit validator (#314)

Author: jacobsimionato

packages/spikes/gulf_genkit_eval/.gitignore

@@ -0,0 +1,24 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# production
+/build
+
+# misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.genkit

packages/spikes/gulf_genkit_eval/GEMINI.md

@@ -0,0 +1,52 @@
+# Gemini Schema Validation Logic
+
+This document outlines the validation rules that should be implemented in the `validateSchema` function. The purpose of this validator is to check for constraints that are not easily expressed in the JSON schema itself, such as conditional requirements and reference integrity.
+
+Each message corresponds to one of the four message schemas: `stream_header.json`, `component_update.json`, `data_model_update.json`, or `begin_rendering.json`.
+
+## `ComponentUpdate` Message Rules
+
+### 1. Component ID Integrity
+
+*   **Uniqueness**: All component `id`s within the `components` array must be unique.
+*   **Reference Validity**: Any property that references a component ID (e.g., `child`, `children`, `entryPointChild`, `contentChild`) must point to an ID that actually exists in the `components` array.
+
+### 2. Component-Specific Property Rules
+
+For each component in the `components` array, the following rules apply based on its `type`:
+
+*   **General**:
+    *   A component must have an `id` and a `type`.
+
+*   **Value Components** (`Heading`, `Text`, `Image`, `Video`, `AudioPlayer`, `TextField`, `CheckBox`, `DateTimeInput`, `MultipleChoice`, `Slider`):
+    *   **Required**: Must have a `value` property.
+
+*   **Container Components** (`Row`, `Column`, `List`):
+    *   **Required**: Must have a `children` property.
+    *   The `children` object must contain *either* `explicitList` *or* `template`, but not both.
+
+*   **Card**:
+    *   **Required**: Must have a `child` property.
+
+*   **Tabs**:
+    *   **Required**: Must have a `tabItems` property, which must be an array.
+    *   Each item in `tabItems` must have a `title` and a `child`.
+
+*   **Modal**:
+    *   **Required**: Must have both `entryPointChild` and `contentChild` properties.
+
+*   **Button**:
+    *   **Required**: Must have `label` and `action` properties.
+
+*   **CheckBox**:
+    *   **Required**: Must have `label` and `value` properties.
+
+## `DataModelUpdate` Message Rules
+
+*   **Path and Contents**: A `DataModelUpdate` message must have a `contents` property. The `path` property is optional.
+*   If `path` is not present, the `contents` object will replace the entire data model.
+*   If `path` is present, the `contents` will be set at that location in the data model.
+
+## `BeginRendering` Message Rules
+
+*   **Root Presence**: Must have a `root` property.

packages/spikes/gulf_genkit_eval/README.md

@@ -0,0 +1,35 @@
+# Genkit Flow
+
+To run the flow, use the following command:
+
+```bash
+npx tsx src/index.ts
+```
+
+## Running a Single Test
+
+You can run the script for a single model and data point by using the `--model` and `--prompt` command-line flags. This is useful for quick tests and debugging.
+
+### Syntax
+
+```bash
+npx tsx src/index.ts --model='<model_name>' --prompt=<prompt_name>
+```
+
+### Example
+
+To run the test with the `gpt-5-nano (reasoning: minimal)` model and the `generateDogUIs` prompt, use the following command:
+
+```bash
+npx tsx src/index.ts --model='gpt-5-nano (reasoning: minimal)' --prompt=generateDogUIs
+```
+
+## Controlling Output
+
+By default, the script only prints the summary table and any errors that occur during generation. To see the full JSON output for each successful generation, use the `--verbose` flag.
+
+### Example
+
+```bash
+npx tsx src/index.ts --verbose
+```

packages/spikes/gulf_genkit_eval/genkit.conf.js

@@ -0,0 +1,10 @@
+import { googleAI } from '@genkit-ai/google-genai';
+import { configure } from 'genkit';
+
+export default configure({
+  plugins: [
+    googleAI(),
+  ],
+  logLevel: 'debug',
+  enableTracingAndMetrics: true,
+});

packages/spikes/gulf_genkit_eval/lib/index.js

@@ -0,0 +1,108 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.componentGeneratorFlow = void 0;
+const google_genai_1 = require("@genkit-ai/google-genai");
+const genkit_1 = require("genkit");
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const openai_1 = require("@genkit-ai/compat-oai/openai");
+const genkitx_anthropic_1 = require("genkitx-anthropic");
+// Read the schema file
+const schemaString = fs.readFileSync(path.join(__dirname, 'schema.json'), 'utf-8');
+const schema = JSON.parse(schemaString);
+const ai = (0, genkit_1.genkit)({
+    plugins: [(0, google_genai_1.googleAI)({ apiKey: process.env.GEMINI_API_KEY }), (0, openai_1.openAI)(), (0, genkitx_anthropic_1.anthropic)({ apiKey: process.env.ANTHROPIC_API_KEY }),],
+});
+// Define a UI component generator flow
+exports.componentGeneratorFlow = ai.defineFlow({
+    name: 'componentGeneratorFlow',
+    inputSchema: genkit_1.z.object({ prompt: genkit_1.z.string(), model: genkit_1.z.any() }),
+    outputSchema: genkit_1.z.any(),
+}, async ({ prompt, model }) => {
+    // Generate structured component data using the schema from the file
+    const { output } = await ai.generate({
+        prompt,
+        model,
+        output: { jsonSchema: schema },
+        // config: {
+        //     thinkingConfig: { thinkingBudget: 0 }
+        // },
+    });
+    if (!output)
+        throw new Error('Failed to generate component');
+    return output;
+});
+// Run the flow
+async function main() {
+    const models = [
+        openai_1.openAI.model('gpt-5-mini'),
+        openai_1.openAI.model('gpt-5'),
+        openai_1.openAI.model('gpt-5-nano'),
+        google_genai_1.googleAI.model('gemini-2.5-flash'),
+        google_genai_1.googleAI.model('gemini-2.5-flash-lite'),
+        genkitx_anthropic_1.claude4Sonnet,
+        genkitx_anthropic_1.claude35Haiku,
+    ];
+    const prompt = `Generate a JSON conforming to the schema to describe the following UI:
+
+A root node has already been created with ID "root". You need to create a ComponentUpdate message now.
+
+A vertical list with:
+Dog breed information
+Dog generator
+
+The dog breed information is a card, which contains a title “Famous Dog breeds”, a header image, and a carousel of different dog breeds. The carousel information should be in the data model at /carousel.
+
+The dog generator is another card which is a form that generates a fictional dog breed with a description
+- Title
+- Description text explaining what it is
+- Dog breed name (text input)
+- Number of legs (number input)
+- Skills (checkboxes)
+- Button called “Generate” which takes the data above and generates a new dog description
+- A divider
+- A section which shows the generated content
+`;
+    for (const model of models) {
+        console.log(`Generating component with model: ${model.name}`);
+        const component = await (0, exports.componentGeneratorFlow)({
+            prompt,
+            model,
+        });
+        console.log(JSON.stringify(component, null, 2));
+    }
+}
+main().catch(console.error);