Skip to content

Commit 0fb786b

Browse files
robinpaul85robinpaul85
committed
Successfully passed genomes to testchat.ts
1 parent 39c7423 commit 0fb786b

File tree

2 files changed

+350
-6
lines changed

2 files changed

+350
-6
lines changed

‎server/src/app.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ export async function launch() {
4444
setAppMiddlewares(app, genomes, doneLoading)
4545

4646
console.log('setting server routes ...')
47-
const routeCallbacks = await setOptionalRoutes(app)
47+
const routeCallbacks = await setOptionalRoutes(app, genomes)
4848
console.log('may set auth routes ...')
4949
/*
5050
!!! the order of middlewares is critical, must be set before data routes !!!
@@ -227,7 +227,7 @@ async function setOptionalRoutes(app) {
227227
for (const fname of serverconfig.routeSetters) {
228228
if (fname.endsWith('.js') || fname.endsWith('.ts')) {
229229
const _ = await import(fname)
230-
const d = _.default(app, basepath)
230+
const d = _.default(app, basepath, genomes)
231231
if (d?.setCloseServer && fname.includes('coverage')) {
232232
routeCallbacks.setCloseServer = d.setCloseServer
233233
}

‎server/src/test/routes/testchat.ts‎

Lines changed: 348 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,364 @@
11
// Test URL: http://localhost:3000/testchat
2-
2+
import path from 'path'
33
import serverconfig from '../../serverconfig.js'
4-
import { test_chatbot_by_dataset } from '../../../routes/chat/test/chatUnitTests.ts'
4+
import { readJSONFile } from '../../../routes/chat/utils.ts'
5+
import { run_chat_pipeline } from '../../../routes/termdb.chat2.ts'
6+
import type {
7+
DEType,
8+
SummaryType,
9+
MatrixType,
10+
SampleScatterType,
11+
FilterTerm,
12+
CategoricalFilterTerm,
13+
NumericFilterTerm
14+
} from '#types'
515

616
process.removeAllListeners('warning')
717

8-
export default function setRoutes(app, basepath) {
18+
const testing = true // This causes raw LLM output to be sent by the agent
19+
const llm = serverconfig.llm
20+
if (!llm) throw 'serverconfig.llm is not configured'
21+
if (llm.provider !== 'SJ' && llm.provider !== 'ollama') {
22+
throw "llm.provider must be 'SJ' or 'ollama'"
23+
}
24+
25+
export default function setRoutes(app, basepath, genomes) {
926
app.get(basepath + '/testchat', async () => {
1027
// (req.res) not currently used
1128
console.log('test chat page')
12-
for (const genome of Object.values(serverconfig.genomes)) {
29+
for (const genome of Object.values(genomes)) {
1330
for (const ds of Object.values((genome as any).datasets)) {
31+
console.log('ds.label:', ds)
1432
if ((ds as any)?.queries?.chat) {
1533
await test_chatbot_by_dataset(ds)
34+
console.log('Tests complete for ' + (ds as any).label)
1635
}
1736
}
1837
}
1938
})
2039
}
40+
41+
export async function test_chatbot_by_dataset(ds: any) {
42+
// Check to see if the dataset supports the AI chatbot
43+
if (!(ds as any)?.queries?.chat.aifiles) throw 'AI dataset JSON file is missing for dataset:' + ds.label
44+
const aifiles = (ds as any)?.queries?.chat.aifiles
45+
const dataset_json = await readJSONFile(aifiles) // Read AI JSON data file
46+
//console.log("dataset_json:", dataset_json)
47+
const aiFilesDir = path.dirname(aifiles)
48+
for (const test_data of dataset_json.TestData) {
49+
//console.log("Test question:", test_data.question)
50+
const test_result = await run_chat_pipeline(
51+
test_data.question,
52+
llm,
53+
serverconfig.aiRoute,
54+
dataset_json,
55+
testing, // This is not needed anymore, need to be deprecated
56+
serverconfig.tpmasterdir + '/' + dataset_json.db,
57+
serverconfig.tpmasterdir + '/' + dataset_json.genedb,
58+
ds,
59+
aiFilesDir
60+
)
61+
console.log('test_result:', test_result)
62+
if (test_result.type == 'html') {
63+
// Resource request
64+
if (test_result.html != test_data.answer) {
65+
console.log(
66+
'html resource request did not match for prompt: ' +
67+
test_data.question +
68+
'. LLM response: ' +
69+
test_result.html +
70+
' Actual response: ' +
71+
test_data.answer
72+
)
73+
}
74+
} else if (test_result.type == 'plot') {
75+
if (test_result.plot == 'summary') {
76+
const validated_llm_summary_output = validate_summary_output(test_result, test_data.answer)
77+
if (!validated_llm_summary_output)
78+
console.log(
79+
'Summary output did not match for prompt: ' +
80+
test_data.question +
81+
'. LLM response: ' +
82+
test_result +
83+
' Actual response: ' +
84+
test_data.answer
85+
)
86+
} else if (test_result.plot == 'dge') {
87+
const validated_llm_DE_output = validate_DE_output(test_result, test_data.answer)
88+
if (!validated_llm_DE_output)
89+
console.log(
90+
'DE output did not match for prompt: ' +
91+
test_data.question +
92+
'. LLM response: ' +
93+
test_result +
94+
' Actual response: ' +
95+
test_data.answer
96+
)
97+
} else if (test_result.plot == 'matrix') {
98+
const validated_llm_matrix_output = validate_matrix_output(test_result, test_data.answer)
99+
if (!validated_llm_matrix_output)
100+
console.log(
101+
'Matrix output did not match for prompt: ' +
102+
test_data.question +
103+
'. LLM response: ' +
104+
test_result +
105+
' Actual response: ' +
106+
test_data.answer
107+
)
108+
} else if (test_result.plot == 'sampleScatter') {
109+
const validated_llm_scatter_output = validate_scatter_output(test_result, test_data.answer)
110+
if (!validated_llm_scatter_output)
111+
console.log(
112+
'SampleScatter output did not match for prompt: ' +
113+
test_data.question +
114+
'. LLM response: ' +
115+
test_result +
116+
' Actual response: ' +
117+
test_data.answer
118+
)
119+
} else {
120+
console.log('Unknown chart type for prompt: ' + test_data.question)
121+
}
122+
} else if (test_result.type == 'resource') {
123+
// Need to add support for resource agent
124+
} else {
125+
console.log('Unknown type for prompt:' + test_data.question)
126+
}
127+
}
128+
}
129+
130+
function validate_summary_output(output: SummaryType, expected: SummaryType): boolean {
131+
if (output.term != expected.term) {
132+
console.log('Summary term did not match. LLM response: ' + output.term + ' Expected: ' + expected.term)
133+
return false
134+
}
135+
136+
if (output.term2 != expected.term2) {
137+
console.log('Summary term2 did not match. LLM response: ' + output.term2 + ' Expected: ' + expected.term2)
138+
return false
139+
}
140+
141+
if (!output.simpleFilter && !expected.simpleFilter) {
142+
return true
143+
}
144+
if (!output.simpleFilter || !expected.simpleFilter) {
145+
console.log(
146+
'Summary simpleFilter mismatch. LLM response: ' +
147+
JSON.stringify(output.simpleFilter) +
148+
' Expected: ' +
149+
JSON.stringify(expected.simpleFilter)
150+
)
151+
return false
152+
}
153+
154+
const filter_valid = validate_filter(output.simpleFilter, expected.simpleFilter)
155+
if (!filter_valid) {
156+
console.log(
157+
'Summary simpleFilter did not match. LLM response: ' +
158+
JSON.stringify(output.simpleFilter) +
159+
' Expected: ' +
160+
JSON.stringify(expected.simpleFilter)
161+
)
162+
}
163+
return filter_valid
164+
}
165+
166+
function validate_DE_output(output_DE_object: DEType, expected_DE_output: DEType): boolean {
167+
let validate_DE_groups = true
168+
if (output_DE_object.group1 && expected_DE_output.group1) {
169+
validate_DE_groups = validate_filter(output_DE_object.group1, expected_DE_output.group1)
170+
} else {
171+
console.log('group1 is missing')
172+
return false
173+
}
174+
175+
if (!validate_DE_groups) console.log('group1 not validated')
176+
177+
if (output_DE_object.group2 && expected_DE_output.group2) {
178+
validate_DE_groups = validate_filter(output_DE_object.group2, expected_DE_output.group2)
179+
} else {
180+
console.log('group2 is missing')
181+
return false
182+
}
183+
if (!validate_DE_groups) console.log('group2 not validated')
184+
185+
return validate_DE_groups
186+
}
187+
188+
function validate_matrix_output(output: MatrixType, expected: MatrixType): boolean {
189+
const outputTerms = output.terms || []
190+
const expectedTerms = expected.terms || []
191+
if (outputTerms.length != expectedTerms.length || !outputTerms.every((t, i) => t == expectedTerms[i])) {
192+
console.log(
193+
'Matrix terms did not match. LLM response: ' +
194+
JSON.stringify(outputTerms) +
195+
' Expected: ' +
196+
JSON.stringify(expectedTerms)
197+
)
198+
return false
199+
}
200+
201+
const outputGenes = output.geneNames || []
202+
const expectedGenes = expected.geneNames || []
203+
if (outputGenes.length != expectedGenes.length || !outputGenes.every((g, i) => g == expectedGenes[i])) {
204+
console.log(
205+
'Matrix geneNames did not match. LLM response: ' +
206+
JSON.stringify(outputGenes) +
207+
' Expected: ' +
208+
JSON.stringify(expectedGenes)
209+
)
210+
return false
211+
}
212+
213+
if (!output.simpleFilter && !expected.simpleFilter) {
214+
return true
215+
}
216+
if (!output.simpleFilter || !expected.simpleFilter) {
217+
console.log(
218+
'Matrix simpleFilter mismatch. LLM response: ' +
219+
JSON.stringify(output.simpleFilter) +
220+
' Expected: ' +
221+
JSON.stringify(expected.simpleFilter)
222+
)
223+
return false
224+
}
225+
226+
const filter_valid = validate_filter(output.simpleFilter, expected.simpleFilter)
227+
if (!filter_valid) {
228+
console.log(
229+
'Matrix simpleFilter did not match. LLM response: ' +
230+
JSON.stringify(output.simpleFilter) +
231+
' Expected: ' +
232+
JSON.stringify(expected.simpleFilter)
233+
)
234+
}
235+
return filter_valid
236+
}
237+
238+
function validate_filter(output_filter: FilterTerm[], expected_filter: FilterTerm[]): boolean {
239+
if (output_filter.length != expected_filter.length) {
240+
return false
241+
} else {
242+
let filter_term_validation = true
243+
for (let i = 0; i < output_filter.length; i++) {
244+
filter_term_validation = validate_each_filter_term(output_filter[i], expected_filter[i]) // Validate each filter term sequentially
245+
if (filter_term_validation == false) {
246+
break
247+
}
248+
}
249+
return filter_term_validation
250+
}
251+
}
252+
253+
function validate_each_filter_term(output_filter_term: FilterTerm, expected_filter_term: FilterTerm): boolean {
254+
if (
255+
(output_filter_term as CategoricalFilterTerm).category &&
256+
(expected_filter_term as CategoricalFilterTerm).category
257+
) {
258+
// Both are categorical filter terms
259+
if (
260+
output_filter_term.term == expected_filter_term.term &&
261+
(output_filter_term as CategoricalFilterTerm).category == (expected_filter_term as CategoricalFilterTerm).category
262+
) {
263+
return compare_join_terms(output_filter_term, expected_filter_term)
264+
} else {
265+
// If term or category fields do not match, fail the test
266+
return false
267+
}
268+
} else if (
269+
output_filter_term.term == expected_filter_term.term &&
270+
(output_filter_term as NumericFilterTerm).start == (expected_filter_term as NumericFilterTerm).start &&
271+
!(output_filter_term as NumericFilterTerm).stop == !(expected_filter_term as NumericFilterTerm).stop
272+
) {
273+
// Numeric filter term when only start term is present
274+
return compare_join_terms(output_filter_term, expected_filter_term)
275+
} else if (
276+
output_filter_term.term == expected_filter_term.term &&
277+
(output_filter_term as NumericFilterTerm).stop == (expected_filter_term as NumericFilterTerm).stop &&
278+
!(output_filter_term as NumericFilterTerm).start == !(expected_filter_term as NumericFilterTerm).start
279+
) {
280+
// Numeric filter term when only stop term is present
281+
return compare_join_terms(output_filter_term, expected_filter_term)
282+
} else if (
283+
output_filter_term.term == expected_filter_term.term &&
284+
(output_filter_term as NumericFilterTerm).start == (expected_filter_term as NumericFilterTerm).start &&
285+
(output_filter_term as NumericFilterTerm).stop == (expected_filter_term as NumericFilterTerm).stop
286+
) {
287+
// Numeric filter term when both start and stop terms are present
288+
return compare_join_terms(output_filter_term, expected_filter_term)
289+
} else {
290+
// Fail in all other conditions such as if one has only a start and the other only a stop
291+
return false
292+
}
293+
}
294+
295+
function compare_join_terms(output_filter_term: FilterTerm, expected_filter_term: FilterTerm): boolean {
296+
if (output_filter_term.join && expected_filter_term.join) {
297+
if (output_filter_term.join == expected_filter_term.join) {
298+
return true
299+
} else {
300+
return false
301+
}
302+
} else if (
303+
(output_filter_term.join && !expected_filter_term.join) ||
304+
(!output_filter_term.join && expected_filter_term.join)
305+
) {
306+
// If one term has a join term while the other is missing, filter term comparison fails
307+
return false
308+
} else {
309+
// If both are missing join terms buth other terms are equal pass the test
310+
return true
311+
}
312+
}
313+
314+
function validate_scatter_output(output: SampleScatterType, expected: SampleScatterType): boolean {
315+
if (output.plotName != expected.plotName) {
316+
console.log(
317+
'SampleScatter plotName did not match. LLM response: ' + output.plotName + ' Expected: ' + expected.plotName
318+
)
319+
return false
320+
}
321+
322+
if (output.colorTW != expected.colorTW) {
323+
console.log(
324+
'SampleScatter colorTW did not match. LLM response: ' + output.colorTW + ' Expected: ' + expected.colorTW
325+
)
326+
return false
327+
}
328+
329+
if (output.shapeTW != expected.shapeTW) {
330+
console.log(
331+
'SampleScatter shapeTW did not match. LLM response: ' + output.shapeTW + ' Expected: ' + expected.shapeTW
332+
)
333+
return false
334+
}
335+
336+
if (output.term0 != expected.term0) {
337+
console.log('SampleScatter term0 did not match. LLM response: ' + output.term0 + ' Expected: ' + expected.term0)
338+
return false
339+
}
340+
341+
if (!output.simpleFilter && !expected.simpleFilter) {
342+
return true
343+
}
344+
if (!output.simpleFilter || !expected.simpleFilter) {
345+
console.log(
346+
'SampleScatter simpleFilter mismatch. LLM response: ' +
347+
JSON.stringify(output.simpleFilter) +
348+
' Expected: ' +
349+
JSON.stringify(expected.simpleFilter)
350+
)
351+
return false
352+
}
353+
354+
const filter_valid = validate_filter(output.simpleFilter, expected.simpleFilter)
355+
if (!filter_valid) {
356+
console.log(
357+
'SampleScatter simpleFilter did not match. LLM response: ' +
358+
JSON.stringify(output.simpleFilter) +
359+
' Expected: ' +
360+
JSON.stringify(expected.simpleFilter)
361+
)
362+
}
363+
return filter_valid
364+
}

0 commit comments

Comments
 (0)