Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

Commit ead1751

Browse files
hcyanghcyang
authored
fix ambiguous duplicate bug (#1269)
* fix ambiguous duplicate bug * add a unit test * make ambugious result more descriptive
1 parent eb38bc0 commit ead1751

File tree

4 files changed

+168
-19
lines changed

4 files changed

+168
-19
lines changed

‎packages/orchestratorlib/src/orchestratorhelper.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@ export class OrchestratorHelper {
286286
throw new Error(`${filePath} has invalid extension - only lu, qna, json, tsv and dispatch files are supported.`);
287287
}
288288

289-
Utility.writeStringToConsoleStdout(`Processing ${filePath}...`);
289+
Utility.writeStringLineToConsoleStdout(`Processing ${filePath}...`);
290290
try {
291291
switch (ext) {
292292
case '.lu':

‎packages/orchestratorlib/src/utility.ts‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4383,12 +4383,20 @@ export class Utility {
43834383
if (predictionStructureWithScore) {
43844384
const predictedScore: number =
43854385
predictionStructureWithScore.predictionScoreStructureFoundation.labelsPredictedScore;
4386+
const labelsIndexes: number[] =
4387+
predictionStructureWithScore.labelsIndexes;
43864388
const scoreArray: number[] =
43874389
predictionStructureWithScore.predictionScoreStructureFoundation.scoreArray;
4390+
UtilityDispatcher.debuggingLog(`generateAmbiguousStatisticsAndHtmlTable<TL>(): predictionStructureWithScore=${UtilityDispatcher.jsonStringify(predictionStructureWithScore)}`);
4391+
UtilityDispatcher.debuggingLog(`generateAmbiguousStatisticsAndHtmlTable<TL>(): scoreArray=${scoreArray}`);
43884392
const scoreArrayAmbiguous: number[][] = scoreArray.map(
4389-
(x: number, index: number) => [x, index, Math.abs((predictedScore - x) / predictedScore)]).filter(
4390-
(x: number[]) => ((x[2] < ambiguousClosenessThreshold) && (x[2] >= 0))).map( // ---- NOTE ---- >= for including the top-score one.
4391-
(x: number[]) => [x[1], x[0], x[2]]);
4393+
(score: number, index: number) => [score, index, Math.abs((predictedScore - score) / predictedScore)]).filter(
4394+
([, index, closenessToPredictedScore]: number[]) =>
4395+
((closenessToPredictedScore < ambiguousClosenessThreshold) && // ---- NOTE ---- closeness smaller than the ambiguous threshold.
4396+
(closenessToPredictedScore >= 0) && // ---- NOTE ---- >=0 for including the top-score one.
4397+
(!labelsIndexes.includes(index)))).map( // ---- NOTE ---- filter out the groud-truth labels.
4398+
([score, index, closenessToPredictedScore]: number[]) => [index, score, closenessToPredictedScore]);
4399+
UtilityDispatcher.debuggingLog(`generateAmbiguousStatisticsAndHtmlTable<TL>(): scoreArrayAmbiguous=${UtilityDispatcher.jsonStringify(scoreArrayAmbiguous)}`);
43924400
if (scoreArrayAmbiguous.length > 0) {
43934401
const labelsScoreStructureHtmlTable: string =
43944402
predictionStructureWithScore.predictionScoreStructureFoundation.predictionScoreStructureFoundationDisplay.labelsScoreStructureHtmlTable;

‎packages/orchestratorlib/test/labelresolver.test.ts‎

Lines changed: 107 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Licensed under the MIT License.
44
*/
55

6-
// import assert = require('assert');
6+
import assert = require('assert');
77

88
import {} from 'mocha';
99

@@ -16,6 +16,8 @@ import {OrchestratorHelper} from '../src/orchestratorhelper';
1616

1717
import {Utility} from '../src/utility';
1818
import {LabelType, Utility as UtilityDispatcher} from '@microsoft/bf-dispatcher';
19+
import {Example} from '@microsoft/bf-dispatcher';
20+
1921
import {UnitTestHelper} from './utility.test';
2022

2123
describe('Test Suite - LabelResolver', () => {
@@ -27,40 +29,131 @@ describe('Test Suite - LabelResolver', () => {
2729
Utility.resetFlagToPrintDebuggingLogToConsole(UnitTestHelper.getDefaultUnitTestDebuggingLogFlag());
2830
this.timeout(UnitTestHelper.getDefaultFunctionalTestTimeout());
2931
const baseModelPath: string = path.resolve('./resources/model/model_dte_bert_3l');
30-
Utility.debuggingLog('Test.0000 LabelResolver.scoreBatch()-Bert-3-layer: downloading a base nerual network language model for unit test');
32+
Utility.debuggingLog('Test.0000 LabelResolver.scoreBatch(): downloading a base nerual network language model for unit test');
3133
await UnitTestHelper.downloadModelFileForTest(
3234
baseModelPath,
3335
OrchestratorBaseModel.defaultHandler,
3436
OrchestratorBaseModel.defaultHandler);
3537
const modelConfig: string = Utility.loadFile(path.resolve(baseModelPath, 'config.json'));
36-
Utility.debuggingLog(`Test.0000 LabelResolver.scoreBatch()-Bert-3-layer: modelConfig=${modelConfig}`);
37-
Utility.debuggingLog(`Test.0000 LabelResolver.scoreBatch()-Bert-3-layer: process.cwd()=${process.cwd()}`);
38+
Utility.debuggingLog(`Test.0000 LabelResolver.scoreBatch(): modelConfig=${modelConfig}`);
39+
Utility.debuggingLog(`Test.0000 LabelResolver.scoreBatch(): process.cwd()=${process.cwd()}`);
3840
const inputPath: string = './resources/data/Columnar/Email_bert.blu';
3941
// -----------------------------------------------------------------------
4042
// ---- NOTE ---- create a LabelResolver object and load the snapshot set.
41-
Utility.debuggingLog('LabelResolver.scoreBatch(), ready to call LabelResolver.createAsync()');
43+
Utility.debuggingLog('Test.0000 LabelResolver.scoreBatch(), ready to call LabelResolver.createAsync()');
4244
await LabelResolver.createAsync(baseModelPath, '');
43-
Utility.debuggingLog('LabelResolver.scoreBatch(), after calling LabelResolver.createAsync()');
45+
Utility.debuggingLog('Test.0000 LabelResolver.scoreBatch(), after calling LabelResolver.createAsync()');
4446
// Utility.debuggingLog('LabelResolver.scoreBatch(), ready to call UtilityLabelResolver.resetLabelResolverSettingUseCompactEmbeddings()');
4547
// UtilityLabelResolver.resetLabelResolverSettingUseCompactEmbeddings(fullEmbeddings);
4648
// Utility.debuggingLog('LabelResolver.scoreBatch(), after calling UtilityLabelResolver.resetLabelResolverSettingUseCompactEmbeddings()');
47-
Utility.debuggingLog('LabelResolver.scoreBatch(), ready to call OrchestratorHelper.getSnapshotFromFile()');
49+
Utility.debuggingLog('Test.0000 LabelResolver.scoreBatch(), ready to call OrchestratorHelper.getSnapshotFromFile()');
4850
const snapshot: Uint8Array = OrchestratorHelper.getSnapshotFromFile(inputPath);
49-
Utility.debuggingLog(`LabelResolver.scoreBatch(): typeof(snapshot)=${typeof snapshot}`);
50-
Utility.debuggingLog(`LabelResolver.scoreBatch(): snapshot.byteLength=${snapshot.byteLength}`);
51-
Utility.debuggingLog('LabelResolver.scoreBatch(), after calling OrchestratorHelper.getSnapshotFromFile()');
52-
Utility.debuggingLog('LabelResolver.scoreBatch(), ready to call LabelResolver.addSnapshot()');
51+
Utility.debuggingLog(`Test.0000 LabelResolver.scoreBatch(): typeof(snapshot)=${typeof snapshot}`);
52+
Utility.debuggingLog(`Test.0000 LabelResolver.scoreBatch(): snapshot.byteLength=${snapshot.byteLength}`);
53+
Utility.debuggingLog('Test.0000 LabelResolver.scoreBatch(), after calling OrchestratorHelper.getSnapshotFromFile()');
54+
Utility.debuggingLog('Test.0000 LabelResolver.scoreBatch(), ready to call LabelResolver.addSnapshot()');
5355
await LabelResolver.addSnapshot(snapshot);
54-
Utility.debuggingLog('LabelResolver.scoreBatch(), after calling LabelResolver.addSnapshot()');
56+
Utility.debuggingLog('Test.0000 LabelResolver.scoreBatch(), after calling LabelResolver.addSnapshot()');
5557
// -----------------------------------------------------------------------
5658
const utterances: string[] = [
5759
'add a flag to it',
5860
'add some more info',
5961
];
6062
// -----------------------------------------------------------------------
6163
const results: any = LabelResolver.scoreBatch(utterances, LabelType.Intent);
62-
UtilityDispatcher.debuggingNamedLog1('LabelResolver.scoreBatch(utterances, LabelType.Intent)', results, 'results');
64+
UtilityDispatcher.debuggingNamedLog1('Test.0000 LabelResolver.scoreBatch(utterances, LabelType.Intent)', results, 'results');
65+
// -----------------------------------------------------------------------
66+
UtilityDispatcher.debuggingLog('THE END - Test.0000 LabelResolver.scoreBatch()');
67+
});
68+
69+
it('Test.0001 LabelResolver.addExample()', async function (): Promise<void> {
70+
const ignore: boolean = UnitTestHelper.getIgnoreFlag();
71+
if (ignore) {
72+
return;
73+
}
74+
Utility.resetFlagToPrintDebuggingLogToConsole(UnitTestHelper.getDefaultUnitTestDebuggingLogFlag());
75+
this.timeout(UnitTestHelper.getDefaultFunctionalTestTimeout());
76+
const baseModelPath: string = path.resolve('./resources/model/model_dte_bert_3l');
77+
Utility.debuggingLog('Test.0001 LabelResolver.addExample(): downloading a base nerual network language model for unit test');
78+
await UnitTestHelper.downloadModelFileForTest(
79+
baseModelPath,
80+
OrchestratorBaseModel.defaultHandler,
81+
OrchestratorBaseModel.defaultHandler,
82+
'pretrained.20200924.microsoft.dte.00.06.en.onnx');
83+
const modelConfig: string = Utility.loadFile(path.resolve(baseModelPath, 'config.json'));
84+
Utility.debuggingLog(`Test.0001 LabelResolver.addExample(): modelConfig=${modelConfig}`);
85+
Utility.debuggingLog(`Test.0001 LabelResolver.addExample(): process.cwd()=${process.cwd()}`);
86+
// -----------------------------------------------------------------------
87+
// ---- NOTE ---- create a LabelResolver object and load the snapshot set.
88+
Utility.debuggingLog('Test.0001 LabelResolver.addExample(), ready to call LabelResolver.createAsync()');
89+
await LabelResolver.createAsync(baseModelPath, '');
90+
Utility.debuggingLog('Test.0001 LabelResolver.addExample(), after calling LabelResolver.createAsync()');
91+
// Utility.debuggingLog('LabelResolver.addExample(), ready to call UtilityLabelResolver.resetLabelResolverSettingUseCompactEmbeddings()');
92+
// UtilityLabelResolver.resetLabelResolverSettingUseCompactEmbeddings(fullEmbeddings);
93+
// Utility.debuggingLog('LabelResolver.addExample(), after calling UtilityLabelResolver.resetLabelResolverSettingUseCompactEmbeddings()');
94+
const example: Example = Example.newIntentExample(
95+
'where is the news on agility',
96+
['Keyword']);
97+
const exampleObject: any = example.toAlternateObject();
98+
Utility.debuggingLog(`Test.0001 LabelResolver.addExample(): exampleObject=${Utility.jsonStringify(exampleObject)}`);
99+
const rvAddExample: any = LabelResolver.addExample(exampleObject);
100+
Utility.debuggingLog(`Test.0001 LabelResolver.addExample(): rv=${rvAddExample}`);
101+
const snapshot: any = LabelResolver.createSnapshot();
102+
Utility.debuggingLog(`Test.0001 LabelResolver.addExample(): snapshot=${snapshot}`);
103+
Utility.debuggingLog(`Test.0001 LabelResolver.addExample(): snapshot.byteLength=${snapshot.byteLength}`);
104+
const snapshotInString: string = new TextDecoder().decode(snapshot);
105+
Utility.debuggingLog(`Test.0001 LabelResolver.addExample(): snapshotInString=${snapshotInString}`);
106+
// -----------------------------------------------------------------------
107+
UtilityDispatcher.debuggingLog('THE END - Test.0001 LabelResolver.addExample()');
108+
});
109+
110+
it('Test.0002 LabelResolver.removeExample()', async function (): Promise<void> {
111+
const ignore: boolean = UnitTestHelper.getIgnoreFlag();
112+
if (ignore) {
113+
return;
114+
}
115+
Utility.resetFlagToPrintDebuggingLogToConsole(UnitTestHelper.getDefaultUnitTestDebuggingLogFlag());
116+
this.timeout(UnitTestHelper.getDefaultFunctionalTestTimeout());
117+
const baseModelPath: string = path.resolve('./resources/model/model_dte_bert_3l');
118+
Utility.debuggingLog('Test.0002 LabelResolver.addExample(): downloading a base nerual network language model for unit test');
119+
await UnitTestHelper.downloadModelFileForTest(
120+
baseModelPath,
121+
OrchestratorBaseModel.defaultHandler,
122+
OrchestratorBaseModel.defaultHandler,
123+
'pretrained.20200924.microsoft.dte.00.06.en.onnx');
124+
const modelConfig: string = Utility.loadFile(path.resolve(baseModelPath, 'config.json'));
125+
Utility.debuggingLog(`Test.0002 LabelResolver.addExample(): modelConfig=${modelConfig}`);
126+
Utility.debuggingLog(`Test.0002 LabelResolver.addExample(): process.cwd()=${process.cwd()}`);
127+
// -----------------------------------------------------------------------
128+
// ---- NOTE ---- create a LabelResolver object and load the snapshot set.
129+
Utility.debuggingLog('Test.0002 LabelResolver.addExample(), ready to call LabelResolver.createAsync()');
130+
await LabelResolver.createAsync(baseModelPath, '');
131+
Utility.debuggingLog('Test.0002 LabelResolver.addExample(), after calling LabelResolver.createAsync()');
132+
// Utility.debuggingLog('LabelResolver.addExample(), ready to call UtilityLabelResolver.resetLabelResolverSettingUseCompactEmbeddings()');
133+
// UtilityLabelResolver.resetLabelResolverSettingUseCompactEmbeddings(fullEmbeddings);
134+
// Utility.debuggingLog('LabelResolver.addExample(), after calling UtilityLabelResolver.resetLabelResolverSettingUseCompactEmbeddings()');
135+
const example: Example = Example.newIntentExample(
136+
'where is the news on agility',
137+
['Keyword']);
138+
const exampleObject: any = example.toAlternateObject();
139+
Utility.debuggingLog(`Test.0002 LabelResolver.addExample(): exampleObject=${Utility.jsonStringify(exampleObject)}`);
140+
const rvAddExample: any = LabelResolver.addExample(exampleObject);
141+
Utility.debuggingLog(`Test.0002 LabelResolver.addExample(): rv=${rvAddExample}`);
142+
const snapshot: any = LabelResolver.createSnapshot();
143+
Utility.debuggingLog(`Test.0002 LabelResolver.addExample(): snapshot=${snapshot}`);
144+
Utility.debuggingLog(`Test.0002 LabelResolver.addExample(): snapshot.byteLength=${snapshot.byteLength}`);
145+
const snapshotInString: string = new TextDecoder().decode(snapshot);
146+
Utility.debuggingLog(`Test.0002 LabelResolver.addExample(): snapshotInString=${snapshotInString}`);
147+
// -----------------------------------------------------------------------
148+
const examples: any = LabelResolver.getExamples();
149+
assert.strictEqual(examples.length, 1);
150+
// -----------------------------------------------------------------------
151+
const rvRemoveExample: any = LabelResolver.removeExample(exampleObject);
152+
Utility.debuggingLog(`Test.0002 LabelResolver.removeExample(): rv=${rvRemoveExample}`);
153+
// -----------------------------------------------------------------------
154+
const examplesAfterRemoveExample: any = LabelResolver.getExamples();
155+
assert.strictEqual(examplesAfterRemoveExample.length, 0);
63156
// -----------------------------------------------------------------------
64-
UtilityDispatcher.debuggingLog('THE END - Test.0000 LabelResolver.runAsync()');
157+
UtilityDispatcher.debuggingLog('THE END - Test.0002 LabelResolver.addExample()');
65158
});
66159
});

‎packages/orchestratorlib/test/utility.test.ts‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,20 @@ import {} from 'mocha';
99

1010
// import * as path from 'path';
1111

12-
import {DictionaryMapUtility} from '@microsoft/bf-dispatcher';
12+
import {DictionaryMapUtility, Span} from '@microsoft/bf-dispatcher';
13+
14+
import {Label} from '@microsoft/bf-dispatcher';
15+
import {LabelType} from '@microsoft/bf-dispatcher';
16+
import {Result} from '@microsoft/bf-dispatcher';
1317

1418
import {ILabelArrayAndMap} from '@microsoft/bf-dispatcher';
19+
import {PredictionStructureWithScoreLabelString} from '@microsoft/bf-dispatcher';
20+
import {PredictionType} from '@microsoft/bf-dispatcher';
1521

1622
import {OrchestratorBaseModel} from '../src/basemodel';
1723
// import {OrchestratorHelper} from '../src/orchestratorhelper';
1824
import {Utility} from '../src/utility';
25+
import {Utility as UtilityDispatcher} from '@microsoft/bf-dispatcher';
1926

2027
// NOTE: "orchestrator_test_3_layer" is an aka.ms alias for the 3 layer model "pretrained.20200924.microsoft.dte.00.03.en.onnx"
2128
// https://aka.ms/orchestrator_test_3_layer === https://aka.ms/pretrained.20200924.microsoft.dte.00.03.en.onnx
@@ -70,6 +77,47 @@ export class UnitTestHelper {
7077
}
7178

7279
describe('Test Suite - utility', () => {
80+
it('Test.0300 Utility.generateAmbiguousStatisticsAndHtmlTable()', function () {
81+
Utility.resetFlagToPrintDebuggingLogToConsole(UnitTestHelper.getDefaultUnitTestDebuggingLogFlag());
82+
this.timeout(UnitTestHelper.getDefaultUnitTestTimeout());
83+
const predictionStructureWithScoreLabelStringArray: PredictionStructureWithScoreLabelString[] = [];
84+
const ambiguousClosenessThreshold: number = 0.2;
85+
const unknownLabelPredictionThreshold: number = 0;
86+
predictionStructureWithScoreLabelStringArray.push(new PredictionStructureWithScoreLabelString(
87+
'hello world', // text: string,
88+
PredictionType.TruePositive, // labelsPredictedEvaluation: number,
89+
['greeting'], // labels: string[],
90+
'greeting', // labelsConcatenated: string,
91+
'', // labelsConcatenatedToHtmlTable: string,
92+
[0], // labelsIndexes: number[],
93+
['greeting'], // labelsPredicted: string[],
94+
'greeting', // labelsPredictedConcatenated: string,
95+
'', // labelsPredictedConcatenatedToHtmlTable: string,
96+
[0], // labelsPredictedIndexes: number[],
97+
0.8, // labelsPredictedScore: number,
98+
['closest-greeting'], // labelsPredictedClosestText: string[],
99+
[new Result(new Label(LabelType.Intent, 'greeting', new Span(0, 0)), 0.8, 'closest-greeting'),
100+
new Result(new Label(LabelType.Intent, 'None', new Span(0, 0)), 0.79, 'closest-None')], // scoreResultArray: Result[],
101+
[0.8, 0.75], // scoreArray: number[],
102+
'', // predictedScoreStructureHtmlTable: string,
103+
'', // labelsScoreStructureHtmlTable: string
104+
));
105+
const scoringAmbiguousResult: {
106+
'scoringAmbiguousUtterancesArrays': string[][];
107+
'scoringAmbiguousUtterancesArraysHtml': string;
108+
'scoringAmbiguousUtteranceSimpleArrays': string[][];
109+
} = Utility.generateAmbiguousStatisticsAndHtmlTable<string>(
110+
predictionStructureWithScoreLabelStringArray,
111+
ambiguousClosenessThreshold,
112+
unknownLabelPredictionThreshold);
113+
UtilityDispatcher.debuggingLog(`scoringAmbiguousUtterancesArraysHtml.length=${scoringAmbiguousResult.scoringAmbiguousUtterancesArraysHtml.length}`);
114+
UtilityDispatcher.debuggingLog(`scoringAmbiguousUtteranceSimpleArrays=${scoringAmbiguousResult.scoringAmbiguousUtteranceSimpleArrays}`);
115+
UtilityDispatcher.debuggingLog(`scoringAmbiguousUtterancesArrays=${scoringAmbiguousResult.scoringAmbiguousUtterancesArrays}`);
116+
assert.strictEqual(scoringAmbiguousResult.scoringAmbiguousUtterancesArraysHtml.length, 818);
117+
assert.strictEqual(scoringAmbiguousResult.scoringAmbiguousUtteranceSimpleArrays.length, 1);
118+
assert.strictEqual(scoringAmbiguousResult.scoringAmbiguousUtterancesArrays.length, 1);
119+
});
120+
73121
it('Test.0200 Utility.buildStringIdNumberValueDictionaryFromStringArray()', function () {
74122
Utility.resetFlagToPrintDebuggingLogToConsole(UnitTestHelper.getDefaultUnitTestDebuggingLogFlag());
75123
this.timeout(UnitTestHelper.getDefaultUnitTestTimeout());

0 commit comments

Comments
 (0)