knowPro: batch validation updates (#1433)

umeshma · web-flow · commit 413c5d35ece2 · 2025-08-07T21:23:56.000Z
Batch testing of scope query schema.
diff --git a/ts/examples/chat/src/memory/knowproTest.ts b/ts/examples/chat/src/memory/knowproTest.ts
@@ -54,6 +54,8 @@ export async function createKnowproTestCommands(
     commands.kpTestChoices = testMultipleChoice;
     commands.kpTestSearch = testSearchScope;
     commands.kpTestScoped = setScoped;
+    commands.kpTestSearchCompare = testSearchCompare;
+    commands.kpTestSearchCompareBatch = testSearchCompareBatch;
 
     async function testHtml(args: string[]) {
         const html = await readAllText(args[0]);
@@ -473,75 +475,118 @@ export async function createKnowproTestCommands(
 
     function testSearchDef(): CommandMetadata {
         const def = searchDef();
-        def.options ??= {};
-        def.options!.compare = argBool("Compare to V1 mode", true);
+        def.description = "Test v2 (scoped) search";
         return def;
     }
     commands.kpTestSearch.metadata = testSearchDef();
     async function testSearchScope(args: string[]) {
         const namedArgs = parseNamedArguments(args, testSearchDef());
-        const queryTranslator =
-            context.conversation! instanceof cm.Memory
-                ? context.conversation!.settings.queryTranslator
-                : context.queryTranslator;
-        const result = namedArgs.compare
-            ? await queryTranslator.translate(namedArgs.query)
-            : undefined;
-        try {
-            context.printer.pushColor(chalk.gray);
-            if (result) {
-                context.printer.writeTranslation(result);
-            }
-        } finally {
-            context.printer.popColor();
-        }
-
-        context.printer.writeHeading("Scope");
+        const queryTranslator = context.getConversationQueryTranslator();
         const resultScope = await queryTranslator.translate2!(namedArgs.query);
         context.printer.writeTranslation(resultScope);
-        if ((result === undefined || result.success) && resultScope.success) {
-            if (result !== undefined) {
-                const error = kpTest.compareSearchQueryScope(
-                    result.data,
-                    resultScope.data,
-                );
-                if (error !== undefined) {
-                    context.printer.writeError(error);
-                }
-            }
-            if (!context.conversation) {
-                return;
-            }
-            const request = parseTypedArguments<kpTest.SearchRequest>(
-                args,
-                searchDef(),
-            );
-            const options: kp.LanguageSearchOptions = {
-                ...kpTest.createSearchOptions(request),
-                compileOptions: {
-                    exactScope: request.exactScope,
-                    applyScope: request.applyScope,
-                },
-            };
-            const query = kp.compileSearchQuery2(
+        if (!resultScope.success || !ensureConversationLoaded()) {
+            return;
+        }
+        const request = parseTypedArguments<kpTest.SearchRequest>(
+            args,
+            searchDef(),
+        );
+        const options: kp.LanguageSearchOptions = {
+            ...kpTest.createSearchOptions(request),
+            compileOptions: {
+                exactScope: request.exactScope,
+                applyScope: request.applyScope,
+            },
+        };
+        const query = kp.compileSearchQuery2(
+            context.conversation!,
+            resultScope.data,
+            options.compileOptions,
+        );
+
+        const results = (
+            await kp.runSearchQueries(context.conversation!, query, options)
+        ).flat();
+        for (let i = 0; i < results.length; ++i) {
+            context.printer.writeConversationSearchResult(
                 context.conversation!,
-                resultScope.data,
-                options.compileOptions,
+                results[i],
+                namedArgs.showKnowledge,
+                namedArgs.showMessages,
+                namedArgs.maxToDisplay,
+                namedArgs.distinct,
             );
+        }
+    }
+
+    commands.kpTestSearchCompare.metadata = testSearchDef();
+    async function testSearchCompare(args: string[]) {
+        const namedArgs = parseNamedArguments(args, testSearchDef());
+        const queryTranslator = context.getConversationQueryTranslator();
+        const result = await kpTest.compareSearchQueryTranslations(
+            queryTranslator,
+            namedArgs.query,
+        );
+        if (!result.success) {
+            context.printer.writeError(result.message);
+            return;
+        }
+        writeSearchComparison(result.data);
+    }
 
-            const results = (
-                await kp.runSearchQueries(context.conversation!, query, options)
-            ).flat();
-            for (let i = 0; i < results.length; ++i) {
-                context.printer.writeConversationSearchResult(
-                    context.conversation!,
-                    results[i],
-                    namedArgs.showKnowledge,
-                    namedArgs.showMessages,
-                    namedArgs.maxToDisplay,
-                    namedArgs.distinct,
+    function compareSearchScopeBatchDef(): CommandMetadata {
+        return {
+            description: "Compare query translations in a batch",
+            args: {
+                srcPath: argSourceFile(),
+            },
+            options: {
+                verbose: argBool("Verbose error output", false),
+            },
+        };
+    }
+    commands.kpTestSearchCompareBatch.metadata = compareSearchScopeBatchDef();
+    async function testSearchCompareBatch(args: string[]) {
+        const namedArgs = parseNamedArguments(args, verifySearchBatchDef());
+        const prevOptions = beginTestBatch(namedArgs);
+        try {
+            const srcPath = namedArgs.srcPath;
+            const results = await kpTest.compareSearchQueryTranslationsBatch(
+                context,
+                srcPath,
+                (result, index, total) => {
+                    context.printer.writeProgress(index + 1, total);
+                    if (result.success) {
+                        const cmp = result.data;
+                        if (cmp.error) {
+                            writeSearchComparison(cmp);
+                        } else {
+                            context.printer.writeLineInColor(
+                                chalk.green,
+                                cmp.query,
+                            );
+                        }
+                    } else {
+                        context.printer.writeError(result.message);
+                    }
+                },
+            );
+            if (!results.success) {
+                context.printer.writeError(results.message);
+                return;
+            }
+            let errorResults = results.data.filter(
+                (r) => r.error !== undefined && r.error.length > 0,
+            );
+            if (errorResults.length > 0) {
+                context.printer.writeLine(`${errorResults.length} errors`);
+                context.printer.writeList(
+                    errorResults.map((e) => e.query),
+                    { type: "ol" },
                 );
             }
+        } finally {
+            endTestBatch(prevOptions);
         }
     }
 
@@ -644,6 +689,15 @@ export async function createKnowproTestCommands(
         }
     }
 
+    function writeSearchComparison(cmp: kpTest.ScopeQueryComparison): void {
+        context.printer.writeJsonInColor(chalk.gray, cmp.expected);
+        context.printer.writeHeading("Scope");
+        context.printer.writeJson(cmp.actual);
+        if (cmp.error) {
+            context.printer.writeError(cmp.error);
+        }
+    }
+
     function beginTestBatch(
         namedArgs: NamedArgs,
     ): kpTest.KnowproContextOptions {
diff --git a/ts/packages/knowProTest/src/common.ts b/ts/packages/knowProTest/src/common.ts
@@ -3,17 +3,23 @@
 
 import { TextEmbeddingModel } from "aiclient";
 import fs from "fs";
-import { ArgDef, NamedArgs, parseCommandLine } from "interactive-app";
+import {
+    ArgDef,
+    getBatchFileLines,
+    NamedArgs,
+    parseCommandLine,
+} from "interactive-app";
 import path from "path";
 import {
     collections,
     dateTime,
     dotProduct,
     generateTextEmbeddingsWithRetry,
     getFileName,
+    writeJsonFile,
 } from "typeagent";
-import { error, Result, Error } from "typechat";
-import { ComparisonResult } from "./types.js";
+import { error, Result, Error, success } from "typechat";
+import { BatchCallback, ComparisonResult } from "./types.js";
 
 export function ensureDirSync(folderPath: string): string {
     if (!fs.existsSync(folderPath)) {
@@ -85,6 +91,40 @@ export async function execCommandLine<T>(
     return error("No args");
 }
 
+export async function runTestBatch<T>(
+    batchFilePath: string,
+    cmdHandler: (cmd: string, args: string[]) => Promise<Result<T>>,
+    destFilePath?: string,
+    cb?: BatchCallback<Result<T>>,
+    stopOnError: boolean = false,
+): Promise<Result<T[]>> {
+    const batchLines = getBatchFileLines(batchFilePath);
+    const results: T[] = [];
+    for (let i = 0; i < batchLines.length; ++i) {
+        const cmd = batchLines[i];
+        const args = getCommandArgs(cmd);
+        if (args.length === 0) {
+            continue;
+        }
+        let response = await cmdHandler(cmd, args);
+        if (!response.success) {
+            response = queryError(cmd, response);
+        }
+        if (cb) {
+            cb(response, i, batchLines.length);
+        }
+        if (response.success) {
+            results.push(response.data);
+        } else if (stopOnError) {
+            return response;
+        }
+    }
+    if (destFilePath) {
+        await writeJsonFile(destFilePath, results);
+    }
+    return success(results);
+}
+
 export function argSourceFile(defaultValue?: string | undefined): ArgDef {
     return {
         description: "Path to source file",
diff --git a/ts/packages/knowProTest/src/knowproContext.ts b/ts/packages/knowProTest/src/knowproContext.ts
@@ -72,6 +72,18 @@ export class KnowproContext {
         return this.conversation!;
     }
 
+    public getConversationQueryTranslator(): kp.SearchQueryTranslator {
+        let queryTranslator = this.queryTranslator;
+
+        if (
+            this.conversation !== undefined &&
+            this.conversation! instanceof cm.Memory
+        ) {
+            queryTranslator = this.conversation!.settings.queryTranslator;
+        }
+        return queryTranslator;
+    }
+
     public createMemorySettings(): cm.MemorySettings {
         return cm.createMemorySettings(64, undefined, this.knowledgeModel);
     }
diff --git a/ts/packages/knowProTest/src/searchTest.ts b/ts/packages/knowProTest/src/searchTest.ts
@@ -4,17 +4,23 @@
 import { KnowproContext } from "./knowproContext.js";
 import { dateTime, readJsonFile, writeJsonFile } from "typeagent";
 import { error, Result, success } from "typechat";
-import { BatchCallback, Comparison } from "./types.js";
+import {
+    BatchCallback,
+    Comparison,
+    SearchRequest,
+    searchRequestDef,
+} from "./types.js";
 import {
     compareNumberArray,
     compareObject,
     compareStringArray,
     dateRangeToTimeRange,
     getCommandArgs,
     queryError,
+    runTestBatch,
 } from "./common.js";
 import { getLangSearchResult } from "./knowproCommands.js";
-import { getBatchFileLines } from "interactive-app";
+import { getBatchFileLines, parseTypedArguments } from "interactive-app";
 import { execSearchRequest } from "./knowproCommands.js";
 import * as kp from "knowpro";
 import { TestRunReport } from "./logging.js";
@@ -34,6 +40,7 @@ export type LangSearchResult = {
     actionMatches?: kp.SemanticRefOrdinal[] | undefined;
 };
 
+// TODO: convert this to use runBatch from common.ts
 export async function runSearchBatch(
     context: KnowproContext,
     batchFilePath: string,
@@ -69,6 +76,7 @@ export async function runSearchBatch(
     }
     return success(results);
 }
+
 async function getSearchResults(
     context: KnowproContext,
     args: string[],
@@ -446,3 +454,61 @@ export function compareSearchExprScope(
     }
     return undefined;
 }
+
+export interface ScopeQueryComparison
+    extends Comparison<
+        kp.querySchema2.SearchQuery,
+        kp.querySchema.SearchQuery
+    > {
+    query: string;
+}
+
+export async function compareSearchQueryTranslations(
+    queryTranslator: kp.SearchQueryTranslator,
+    query: string,
+): Promise<Result<ScopeQueryComparison>> {
+    if (!queryTranslator.translate2) {
+        return error("not supported");
+    }
+    const result = await queryTranslator.translate(query);
+    if (!result.success) {
+        return result;
+    }
+    const resultScope = await queryTranslator.translate2!(query);
+    if (!resultScope.success) {
+        return resultScope;
+    }
+
+    let cmp: ScopeQueryComparison = {
+        actual: resultScope.data,
+        expected: result.data,
+        query,
+    };
+    cmp.error = compareSearchQueryScope(result.data, resultScope.data);
+    return success(cmp);
+}
+
+export async function compareSearchQueryTranslationsBatch(
+    context: KnowproContext,
+    batchFilePath: string,
+    cb?: BatchCallback<Result<ScopeQueryComparison>>,
+    stopOnError: boolean = false,
+): Promise<Result<ScopeQueryComparison[]>> {
+    const queryTranslator = context.getConversationQueryTranslator();
+    return runTestBatch(
+        batchFilePath,
+        (_, args: string[]) => {
+            const request = parseTypedArguments<SearchRequest>(
+                args,
+                searchRequestDef(),
+            );
+            return compareSearchQueryTranslations(
+                queryTranslator,
+                request.query,
+            );
+        },
+        undefined,
+        cb,
+        stopOnError,
+    );
+}
diff --git a/ts/packages/knowProTest/src/types.ts b/ts/packages/knowProTest/src/types.ts
diff --git a/ts/packages/textPro/README.md b/ts/packages/textPro/README.md
