Skip to content

Commit f565280

Browse files
umeshmaumeshma
authored
Memory updates (#1427)
* ReadMe update * v2 schema consistency with v1 * Updates to testing commands to batch compare v1 and v2 (scoped) query
1 parent 5b3ebba commit f565280

File tree

7 files changed

+112
-40
lines changed

7 files changed

+112
-40
lines changed

‎ts/examples/chat/src/memory/knowproTest.ts

Lines changed: 57 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ import {
77
argNum,
88
CommandHandler,
99
CommandMetadata,
10+
makeArg,
11+
NamedArgs,
1012
parseNamedArguments,
1113
parseTypedArguments,
1214
ProgressBar,
@@ -51,6 +53,7 @@ export async function createKnowproTestCommands(
5153
commands.kpTestHtmlParts = testHtmlParts;
5254
commands.kpTestChoices = testMultipleChoice;
5355
commands.kpTestSearch = testSearchScope;
56+
commands.kpTestScoped = setScoped;
5457

5558
async function testHtml(args: string[]) {
5659
const html = await readAllText(args[0]);
@@ -197,10 +200,9 @@ export async function createKnowproTestCommands(
197200
if (!ensureConversationLoaded()) {
198201
return;
199202
}
203+
const namedArgs = parseNamedArguments(args, searchBatchDef());
204+
const prevOptions = beginTestBatch(namedArgs);
200205
try {
201-
beginTestBatch();
202-
203-
const namedArgs = parseNamedArguments(args, searchBatchDef());
204206
const destPath =
205207
namedArgs.destPath ??
206208
changeFileExt(namedArgs.srcPath, ".json", "_results");
@@ -224,7 +226,7 @@ export async function createKnowproTestCommands(
224226
return;
225227
}
226228
} finally {
227-
endTestBatch();
229+
endTestBatch(prevOptions);
228230
}
229231
}
230232

@@ -244,10 +246,9 @@ export async function createKnowproTestCommands(
244246
if (!ensureConversationLoaded()) {
245247
return;
246248
}
249+
const namedArgs = parseNamedArguments(args, verifySearchBatchDef());
250+
const prevOptions = beginTestBatch(namedArgs);
247251
try {
248-
beginTestBatch();
249-
250-
const namedArgs = parseNamedArguments(args, verifySearchBatchDef());
251252
const srcPath = namedArgs.srcPath;
252253

253254
const startTimestamp = new Date();
@@ -273,7 +274,7 @@ export async function createKnowproTestCommands(
273274
startTimestamp,
274275
);
275276
} finally {
276-
endTestBatch();
277+
endTestBatch(prevOptions);
277278
}
278279
}
279280

@@ -293,10 +294,9 @@ export async function createKnowproTestCommands(
293294
if (!ensureConversationLoaded()) {
294295
return;
295296
}
297+
const namedArgs = parseNamedArguments(args, answerBatchDef());
298+
const prevOptions = beginTestBatch(namedArgs);
296299
try {
297-
beginTestBatch();
298-
299-
const namedArgs = parseNamedArguments(args, answerBatchDef());
300300
const srcPath = namedArgs.srcPath;
301301
const destPath =
302302
namedArgs.destPath ??
@@ -321,7 +321,7 @@ export async function createKnowproTestCommands(
321321
},
322322
);
323323
} finally {
324-
endTestBatch();
324+
endTestBatch(prevOptions);
325325
}
326326
}
327327

@@ -337,6 +337,10 @@ export async function createKnowproTestCommands(
337337
0.9,
338338
),
339339
verbose: argBool("Verbose error output", false),
340+
scoped: argBool(
341+
"Translate NL queries into scoped search expressions",
342+
false,
343+
),
340344
},
341345
};
342346
}
@@ -345,10 +349,9 @@ export async function createKnowproTestCommands(
345349
if (!ensureConversationLoaded()) {
346350
return;
347351
}
352+
const namedArgs = parseNamedArguments(args, verifyAnswerBatchDef());
353+
const prevOptions = beginTestBatch(namedArgs);
348354
try {
349-
beginTestBatch();
350-
351-
const namedArgs = parseNamedArguments(args, verifyAnswerBatchDef());
352355
const minSimilarity = namedArgs.similarity;
353356
const srcPath = namedArgs.srcPath;
354357

@@ -374,7 +377,7 @@ export async function createKnowproTestCommands(
374377
context.printer.writeError(results.message);
375378
}
376379
} finally {
377-
endTestBatch();
380+
endTestBatch(prevOptions);
378381
}
379382
}
380383

@@ -471,7 +474,7 @@ export async function createKnowproTestCommands(
471474
function testSearchDef(): CommandMetadata {
472475
const def = searchDef();
473476
def.options ??= {};
474-
def.options!.compare = argBool("Compare to V1 mode", false);
477+
def.options!.compare = argBool("Compare to V1 mode", true);
475478
return def;
476479
}
477480
commands.kpTestSearch.metadata = testSearchDef();
@@ -484,8 +487,13 @@ export async function createKnowproTestCommands(
484487
const result = namedArgs.compare
485488
? await queryTranslator.translate(namedArgs.query)
486489
: undefined;
487-
if (result) {
488-
context.printer.writeTranslation(result);
490+
try {
491+
context.printer.pushColor(chalk.gray);
492+
if (result) {
493+
context.printer.writeTranslation(result);
494+
}
495+
} finally {
496+
context.printer.popColor();
489497
}
490498

491499
context.printer.writeHeading("Scope");
@@ -501,7 +509,7 @@ export async function createKnowproTestCommands(
501509
context.printer.writeError(error);
502510
}
503511
}
504-
if (!ensureConversationLoaded()) {
512+
if (!context.conversation) {
505513
return;
506514
}
507515
const request = parseTypedArguments<kpTest.SearchRequest>(
@@ -537,6 +545,26 @@ export async function createKnowproTestCommands(
537545
}
538546
}
539547

548+
function setScopedDef(): CommandMetadata {
549+
return {
550+
description: "Enable/disable scoped search",
551+
options: {
552+
enable: makeArg("Enable scoped search", "boolean", undefined),
553+
},
554+
};
555+
}
556+
commands.kpTestScoped.metadata = setScopedDef();
557+
async function setScoped(args: string[]): Promise<void> {
558+
const namedArgs = parseNamedArguments(args, setScopedDef());
559+
if (namedArgs.enable !== undefined) {
560+
context.options.scopedSearch = namedArgs.enable;
561+
} else {
562+
context.printer.writeLine(
563+
`Scoped search: ${context.options.scopedSearch}`,
564+
);
565+
}
566+
}
567+
540568
function writeSearchScore(
541569
result: kpTest.Comparison<kpTest.LangSearchResults>,
542570
verbose: boolean = false,
@@ -616,12 +644,17 @@ export async function createKnowproTestCommands(
616644
}
617645
}
618646

619-
function beginTestBatch(): void {
620-
context.retryNoAnswer = true;
647+
function beginTestBatch(
648+
namedArgs: NamedArgs,
649+
): kpTest.KnowproContextOptions {
650+
const prevOptions = { ...context.options };
651+
context.options.retryNoAnswer = true;
652+
return prevOptions;
621653
}
622654

623-
function endTestBatch(): void {
624-
context.retryNoAnswer = false;
655+
function endTestBatch(savedOptions: kpTest.KnowproContextOptions): void {
656+
context.options = savedOptions;
625657
}
658+
626659
return;
627660
}

‎ts/examples/examplesLib/src/chalkWriter.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,6 @@ export class ChalkWriter extends ConsoleWriter {
193193
}
194194

195195
public writeTranslation<T>(result: Result<T>) {
196-
this.writeLine();
197196
if (result.success) {
198197
this.writeJson(result.data);
199198
} else {

‎ts/packages/knowPro/src/searchQuerySchema_v2.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,18 +12,18 @@ export type FacetTerm = {
1212

1313
// Use to find information about specific, tangible people, places, institutions or things only..
1414
// This includes entities with particular facets
15-
// Abstract concepts or topics are not entities.
15+
// Abstract concepts or topics are not entityTerms. Use string for them
1616
// Any terms will match fuzzily.
1717
export type EntityTerm = {
1818
// the name of the entity or thing such as "Bach", "Great Gatsby", "frog" or "piano" or "we", "I"; "*" means match any entity name
1919
name: string;
2020
isNamePronoun: boolean;
2121
// the specific types of the entity such as "book", "movie", "song", "speaker", "person", "artist", "animal", "instrument", "school", "room", "museum", "food" etc.
22-
// Do not include generic types such as: "entity", "object", "thing", "concept" etc.
22+
// Generic types like "object", "thing" etc. are NOT allowed
2323
// An entity can have multiple types; entity types should be single words
2424
type?: string[];
2525
// Facet terms search for properties or attributes of the entity.
26-
// Eg: color(blue), profession(writer), author(*), aunt(Agatha), weight(4kg), phoneNumber(...), title(*) etc.
26+
// Eg: color(blue), profession(writer), author(*), aunt(Agatha), weight(4kg), phoneNumber(...), etc.
2727
facets?: FacetTerm[];
2828
};
2929

@@ -65,7 +65,7 @@ export type SearchFilter = {
6565
actionSearchTerm?: ActionTerm;
6666
entitySearchTerms?: EntityTerm[];
6767
// searchTerms:
68-
// Use for all concepts, topics, or other search terms that don't fit ActionTerms or EntityTerms
68+
// Concepts, topics or other terms that don't fit ActionTerms or EntityTerms
6969
// - Do not use noisy searchTerms like "topic", "topics", "subject", "discussion" etc. even if they are mentioned in the user request
7070
// - Phrases like 'email address' or 'first name' are a single term
7171
// - use empty searchTerms array when use asks for summaries

‎ts/packages/knowProTest/src/knowproCommands.ts

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,11 @@ export async function execSearchRequest(
105105
debugContext.searchQueryExpr = compiledQueries;
106106
searchResults = success(queryResults.flat());
107107
} else {
108-
if (request.scoped === undefined || request.scoped === false) {
108+
let scoped = context.options.scopedSearch;
109+
if (request.scoped !== undefined) {
110+
scoped = request.scoped;
111+
}
112+
if (!scoped) {
109113
//
110114
// Run raw NLP query
111115
//
@@ -191,7 +195,7 @@ export async function execGetAnswerRequest(
191195
request,
192196
searchResults.data,
193197
progressCallback,
194-
context.retryNoAnswer,
198+
context.options.retryNoAnswer,
195199
);
196200
response.answerResponses = answerResponses;
197201
return response;

‎ts/packages/knowProTest/src/knowproContext.ts

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,11 @@ import path from "path";
1010
import { createEmbeddingCache } from "knowledge-processor";
1111
import { PromptSection } from "typechat";
1212

13+
export type KnowproContextOptions = {
14+
retryNoAnswer: boolean;
15+
scopedSearch: boolean;
16+
};
17+
1318
export class KnowproContext {
1419
public knowledgeModel: ChatModel;
1520
public similarityModel: TextEmbeddingModel;
@@ -18,8 +23,11 @@ export class KnowproContext {
1823
public queryTranslator: kp.SearchQueryTranslator;
1924
public answerGenerator: kp.AnswerGenerator;
2025
public termParser: cm.SearchTermParser;
21-
public retryNoAnswer: boolean;
2226
public log: KnowproLog;
27+
//
28+
// Global options and overrides
29+
//
30+
public options: KnowproContextOptions;
2331

2432
public tokenStats: openai.CompletionUsageStats;
2533
public promptHandler?:
@@ -42,13 +50,19 @@ export class KnowproContext {
4250
this.answerGenerator = new kp.AnswerGenerator(
4351
kp.createAnswerGeneratorSettings(this.knowledgeModel),
4452
);
45-
this.retryNoAnswer = false;
4653
this.termParser = new cm.SearchTermParser();
4754
this.tokenStats = {
4855
completion_tokens: 0,
4956
prompt_tokens: 0,
5057
total_tokens: 0,
5158
};
59+
//
60+
// Set default global options and overrides
61+
//
62+
this.options = {
63+
retryNoAnswer: false,
64+
scopedSearch: false,
65+
};
5266
}
5367

5468
public ensureConversationLoaded(): kp.IConversation {

‎ts/packages/knowProTest/src/types.ts

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,13 @@
11
// Copyright (c) Microsoft Corporation.
22
// Licensed under the MIT License.
33

4-
import { CommandMetadata, arg, argBool, argNum } from "interactive-app";
4+
import {
5+
CommandMetadata,
6+
arg,
7+
argBool,
8+
argNum,
9+
makeArg,
10+
} from "interactive-app";
511
import * as kp from "knowpro";
612
import { Result } from "typechat";
713
import { argSourceFile } from "./common.js";
@@ -53,9 +59,10 @@ export function searchRequestDef(): CommandMetadata {
5359
"Thread description: scope matches to the thread best matching this description",
5460
),
5561
when: arg("Sub-query to scope matches (early experimental)"),
56-
scoped: argBool(
62+
scoped: makeArg(
5763
"Translate NL queries into scoped search expressions",
58-
false,
64+
"boolean",
65+
undefined,
5966
),
6067
},
6168
};

‎ts/packages/textPro/README.md

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,26 @@
11
# TextPro
22

3-
(Work in progress)
3+
TextPro is **sample code** for parsing, converting and manipulating text data and documents.
44

5-
TextPro is **sample library code** for parsing and manipulating text data and documents.
5+
TextPro uses markdown as in intermediate format. Other formats like html are converted (and simplified) into markdown.
66

7-
- MD
8-
- Html
7+
TextPro analyzes markdown documents and extracts:
8+
9+
- Document blocks/chunks such as headings, lists, tables, links, images, etc.
10+
- Inferred knowledge such as entities, topics structured tags etc. This knowledge can be inferred from markdown information without using an LLM.
11+
TextPro is also used by Document memory
12+
13+
TextPro is currently used by [Document Memory](../memory//conversation/src/docImport.ts) to import html and markdown documents.
14+
15+
## Module Overview
16+
17+
- [markdown](./src/markdown.ts)
18+
- Markdown parsing and analysis
19+
- Markdown chunking, including splitting large lists and tables
20+
- [html](./src/html.ts)
21+
- html to markdown conversion
22+
- html simplification
23+
- html to text
924

1025
## Trademarks
1126

0 commit comments

Comments
 (0)