Replacing html type with text in all agents save for the resourceagent for the pre-approved html resources

Author: compbiolover

server/routes/chat/DEagent.ts

@@ -96,23 +96,23 @@ export async function extract_DE_search_terms_from_query(
 			return await validate_DE_response(response, ds, dataset_db_output.db_rows)
 		}
 	} else {
-		return { type: 'html', html: 'Differential gene expression not supported for this dataset' }
+		return { type: 'text', text: 'Differential gene expression not supported for this dataset' }
 	}
 }
 
 async function validate_DE_response(response: string, ds: any, db_rows: DbRows[]) {
 	const response_type = safeParseLlmJson(response)
-	let html = ''
+	let text = ''
 	let group1: any
 	let samples1lst: any
 	const name1 = generate_group_name(response_type.group1, db_rows)
 	if (!response_type.group1) {
-		html += 'group1 not present in DE output'
+		text += 'group1 not present in DE output'
 	} else {
 		// Validate filter terms
 		const validated_filters = validate_filter(response_type.group1, ds, name1)
-		if (validated_filters.html.length > 0) {
-			html += validated_filters.html
+		if (validated_filters.text.length > 0) {
+			text += validated_filters.text
 		} else {
 			const samples1 = await get_samples({ filter: validated_filters.simplefilter }, ds, true) // true is to bypass permission check
 			samples1lst = samples1.map((item: any) => ({
@@ -130,11 +130,11 @@ async function validate_DE_response(response: string, ds: any, db_rows: DbRows[]
 	let samples2lst: any
 	const name2 = generate_group_name(response_type.group2, db_rows)
 	if (!response_type.group2) {
-		html += 'group2 not present in DE output'
+		text += 'group2 not present in DE output'
 	} else {
 		const validated_filters = validate_filter(response_type.group2, ds, name2)
-		if (validated_filters.html.length > 0) {
-			html += validated_filters.html
+		if (validated_filters.text.length > 0) {
+			text += validated_filters.text
 		} else {
 			const samples2 = await get_samples({ filter: validated_filters.simplefilter }, ds, true) // true is to bypass permission check
 			samples2lst = samples2.map((item: any) => ({
@@ -155,16 +155,16 @@ async function validate_DE_response(response: string, ds: any, db_rows: DbRows[]
 		if (response_type.method == 'edgeR' || response_type.method == 'limma' || response_type.method == 'wilcoxon') {
 			settings = { volcano: { method: response_type.method } }
 		} else {
-			html += 'Unknown DE method: ' + response_type.method
+			text += 'Unknown DE method: ' + response_type.method
 		}
 	}
 
-	if (html.length > 0) {
-		html = removeLastOccurrence(
-			html,
+	if (text.length > 0) {
+		text = removeLastOccurrence(
+			text,
 			'For now, the maximum number of filter terms supported through the chatbot is ' + num_filter_cutoff // Remove duplicated statements in error message
 		)
-		return { type: 'html', html: html }
+		return { type: 'text', text }
 	} else {
 		const pp_plot_json: any = { childType: 'volcano', termType: 'geneExpression', chartType: 'differentialAnalysis' }
 		const groups = [group1, group2]

server/routes/chat/filter.ts

@@ -51,9 +51,9 @@ export const FILTER_DESCRIPTION =
 	'"stop" an optional filter which is defined when a higher cutoff is defined in the user input for the numeric variable. '
 
 function sortSameCategoricalFilterKeys(filters: any[], ds: any): any {
-	let html = ''
+	let text = ''
 	const keys = filters.map(f => f.term)
-	if (new Set(keys).size == keys.length) return { filters: filters, html: html } // All filter terms have separate keys
+	if (new Set(keys).size == keys.length) return { filters: filters, text } // All filter terms have separate keys
 
 	const seen = new Set<string>()
 	const categorical_filter_terms_with_multiple_fields = new Set<string>()
@@ -70,7 +70,7 @@ function sortSameCategoricalFilterKeys(filters: any[], ds: any): any {
 			ds.cohort.termdb.q.termjsonByOneid(key.toUpperCase()) ||
 			ds.cohort.termdb.q.termjsonByOneid(key.toLowerCase())
 		if (!term) {
-			html += 'invalid filter id:' + key
+			text += 'invalid filter id:' + key
 		} else {
 			if (term.type == 'categorical') {
 				const multiple_fields = filters.filter(x => x.term == key)
@@ -97,29 +97,29 @@ function sortSameCategoricalFilterKeys(filters: any[], ds: any): any {
 			}
 		}
 	}
-	return { filters: sorted_filter, html: html }
+	return { filters: sorted_filter, text }
 }
 
 export function validate_filter(filters: FilterTerm[], ds: any, group_name: string): any {
 	if (!Array.isArray(filters)) throw 'filter is not array'
 	const sorted_filters = sortSameCategoricalFilterKeys(filters, ds)
-	let filter_result: { simplefilter?: any; html: string } = { html: sorted_filters.html }
+	let filter_result: { simplefilter?: any; text: string } = { text: sorted_filters.text }
 	if (sorted_filters.filters.length <= 2) {
 		// If number of filter terms <=2 then simply a single iteration of generate_filter_term() is sufficient
 		const generated = generate_filter_term(sorted_filters.filters, ds)
 		filter_result.simplefilter = generated.simplefilter
-		filter_result.html += generated.html
+		filter_result.text += generated.text
 	} else {
 		if (sorted_filters.filters.length > num_filter_cutoff) {
-			filter_result.html +=
+			filter_result.text +=
 				'For now, the maximum number of filter terms supported through the chatbot is ' + num_filter_cutoff
 			if (group_name.length > 0) {
 				// Group name is blank for summary filter, this is case for groups
-				filter_result.html +=
+				filter_result.text +=
 					' . The number of filter terms for group ' + group_name + ' is ' + sorted_filters.filters.length + '\n' // Added temporary logic to restrict the number of filter terms to num_filter_cutoff.
 			} else {
 				// For summary filter prompts which do not have a group
-				filter_result.html += 'The number of filter terms for this query is ' + sorted_filters.filters.length
+				filter_result.text += 'The number of filter terms for this query is ' + sorted_filters.filters.length
 			}
 		} else {
 			// When number of filter terms is greater than 2, then in each iteration the first two terms are taken and a filter object is created which is passed in the following iteration as a filter term
@@ -135,11 +135,11 @@ export function validate_filter(filters: FilterTerm[], ds: any, group_name: stri
 			}
 		}
 	}
-	return { simplefilter: filter_result.simplefilter, html: filter_result.html }
+	return { simplefilter: filter_result.simplefilter, text: filter_result.text }
 }
 
 function generate_filter_term(filters: any, ds: any) {
-	let invalid_html = ''
+	let invalid_text = ''
 	const localfilter: any = { type: 'tvslst', in: true, lst: [] as any[] }
 	for (const f of filters) {
 		if (f.type == 'tvslst') {
@@ -150,7 +150,7 @@ function generate_filter_term(filters: any, ds: any) {
 				ds.cohort.termdb.q.termjsonByOneid(f.term.toUpperCase()) ||
 				ds.cohort.termdb.q.termjsonByOneid(f.term.toLowerCase())
 			if (!term) {
-				invalid_html += 'invalid filter id:' + f.term
+				invalid_text += 'invalid filter id:' + f.term
 			} else {
 				if (f.join) {
 					localfilter.join = f.join
@@ -164,7 +164,7 @@ function generate_filter_term(filters: any, ds: any) {
 							if (ck.toLowerCase() == category.toLowerCase()) cat = ck
 							else if (term.values[ck].label?.toLowerCase() == category.toLowerCase()) cat = ck
 						}
-						if (!cat) invalid_html += 'invalid category from ' + JSON.stringify({ term: f.term, category })
+						if (!cat) invalid_text += 'invalid category from ' + JSON.stringify({ term: f.term, category })
 						else values.push({ key: cat })
 					}
 					localfilter.lst.push({
@@ -193,7 +193,7 @@ function generate_filter_term(filters: any, ds: any) {
 						range.start = Number(f.start)
 						range.stop = Number(f.stop)
 					} else {
-						invalid_html += 'Neither greater or lesser defined'
+						invalid_text += 'Neither greater or lesser defined'
 					}
 					numeric.tvs.ranges.push(range)
 					localfilter.lst.push(numeric)
@@ -203,7 +203,7 @@ function generate_filter_term(filters: any, ds: any) {
 	}
 	if (filters.length > 1 && !localfilter.join) {
 		localfilter.join = 'and' // Hardcoding and when the LLM is not able to detect the connection
-		//invalid_html += 'Connection (and/or) between the filter terms is not clear, please try to rephrase your question'
+		//invalid_text += 'Connection (and/or) between the filter terms is not clear, please try to rephrase your question'
 	}
-	return { simplefilter: localfilter, html: invalid_html }
+	return { simplefilter: localfilter, text: invalid_text }
 }

server/routes/chat/matrixagent.ts

@@ -79,16 +79,16 @@ export async function extract_matrix_search_terms_from_query(
 function validate_matrix_response(response: string, common_genes: string[], dataset_json: any, ds: any) {
 	const response_type = safeParseLlmJson(response)
 	const pp_plot_json: any = { chartType: 'matrix' }
-	let html = ''
+	let text = ''
 
-	if (response_type.html) html = response_type.html
+	if (response_type.text) text = response_type.text
 
 	// Must have at least one of terms or geneNames
 	if (
 		(!response_type.terms || response_type.terms.length == 0) &&
 		(!response_type.geneNames || response_type.geneNames.length == 0)
 	) {
-		html += 'At least one clinical term or gene name is required for a matrix plot'
+		text += 'At least one clinical term or gene name is required for a matrix plot'
 	}
 
 	// Validate dictionary terms — use shorthand { id } at tw top level
@@ -97,7 +97,7 @@ function validate_matrix_response(response: string, common_genes: string[], data
 		for (const t of response_type.terms) {
 			const term: any = ds.cohort.termdb.q.termjsonByOneid(t)
 			if (!term) {
-				html += 'invalid term id:' + t + ' '
+				text += 'invalid term id:' + t + ' '
 			} else {
 				twLst.push({ id: term.id })
 			}
@@ -110,7 +110,7 @@ function validate_matrix_response(response: string, common_genes: string[], data
 		for (const g of response_type.geneNames) {
 			const gene_hits = common_genes.filter(gene => gene == g.toLowerCase())
 			if (gene_hits.length == 0) {
-				html += 'invalid gene name:' + g + ' '
+				text += 'invalid gene name:' + g + ' '
 			} else {
 				const geneName = g.toUpperCase()
 				if (dataset_json.hasGeneExpression) {
@@ -125,15 +125,15 @@ function validate_matrix_response(response: string, common_genes: string[], data
 	// Validate filters
 	if (response_type.simpleFilter && response_type.simpleFilter.length > 0) {
 		const validated_filters = validate_filter(response_type.simpleFilter, ds, '')
-		if (validated_filters.html.length > 0) {
-			html += validated_filters.html
+		if (validated_filters.text.length > 0) {
+			text += validated_filters.text
 		} else {
 			pp_plot_json.filter = validated_filters.simplefilter
 		}
 	}
 
-	if (html.length > 0) {
-		return { type: 'html', html: html }
+	if (text.length > 0) {
+		return { type: 'text', text }
 	} else {
 		// Structure as termgroups matching what matrix.js expects:
 		// termgroups: [{ name: '', lst: [ { term: {...} }, ... ] }]

server/routes/chat/samplescatteragent.ts

@@ -13,7 +13,7 @@ export async function extract_samplescatter_terms_from_query(
 	testing: boolean
 ) {
 	if (!dataset_json.prebuiltPlots || dataset_json.prebuiltPlots.length == 0) {
-		return { type: 'html', html: 'No pre-built scatter plots (t-SNE/UMAP) are available for this dataset' }
+		return { type: 'text', text: 'No pre-built scatter plots (t-SNE/UMAP) are available for this dataset' }
 	}
 
 	const Schema = {
@@ -99,20 +99,20 @@ export async function extract_samplescatter_terms_from_query(
 
 function validate_samplescatter_response(response: string, common_genes: string[], dataset_json: any, ds: any) {
 	const response_type = safeParseLlmJson(response)
-	let html = ''
+	let text = ''
 
-	if (response_type.html) html = response_type.html
+	if (response_type.text) text = response_type.text
 
 	// Validate plotName against prebuiltPlots
 	if (!response_type.plotName) {
-		html += 'plotName is required for sample scatter output'
+		text += 'plotName is required for sample scatter output'
 	} else {
 		const matchedPlot = dataset_json.prebuiltPlots.find(
 			(p: any) => p.name.toLowerCase() == response_type.plotName.toLowerCase()
 		)
 		if (!matchedPlot) {
 			const availablePlots = dataset_json.prebuiltPlots.map((p: any) => p.name).join(', ')
-			html += 'Unknown plot name: ' + response_type.plotName + '. Available plots are: ' + availablePlots
+			text += 'Unknown plot name: ' + response_type.plotName + '. Available plots are: ' + availablePlots
 		}
 	}
 
@@ -133,8 +133,8 @@ function validate_samplescatter_response(response: string, common_genes: string[
 			return
 		}
 		const termValidation = validate_term(termName, common_genes, dataset_json, ds)
-		if (termValidation.html.length > 0) {
-			html += termValidation.html
+		if (termValidation.text.length > 0) {
+			text += termValidation.text
 		} else {
 			const tw: any = { ...termValidation.term_type }
 			if (termValidation.category == 'float' || termValidation.category == 'integer') {
@@ -151,15 +151,15 @@ function validate_samplescatter_response(response: string, common_genes: string[
 	// Validate filters
 	if (response_type.simpleFilter && response_type.simpleFilter.length > 0) {
 		const validated_filters = validate_filter(response_type.simpleFilter, ds, '')
-		if (validated_filters.html.length > 0) {
-			html += validated_filters.html
+		if (validated_filters.text.length > 0) {
+			text += validated_filters.text
 		} else {
 			pp_plot_json.filter = validated_filters.simplefilter
 		}
 	}
 
-	if (html.length > 0) {
-		return { type: 'html', html: html }
+	if (text.length > 0) {
+		return { type: 'text', text }
 	} else {
 		return { type: 'plot', plot: pp_plot_json }
 	}

server/routes/chat/summaryagent.ts

@@ -123,16 +123,16 @@ export async function extract_summary_terms(
 function validate_summary_response(response: string, common_genes: string[], dataset_json: any, ds: any) {
 	const response_type = safeParseLlmJson(response)
 	const pp_plot_json: any = { chartType: 'summary' }
-	let html = ''
-	if (response_type.html) html = response_type.html
+	let text = ''
+	if (response_type.text) text = response_type.text
 	if (!response_type.term) {
-		html += 'term type is not present in summary output'
-		return { type: 'html', html: html }
+		text += 'term type is not present in summary output'
+		return { type: 'text', text }
 	}
 	const term1_validation = validate_term(response_type.term, common_genes, dataset_json, ds)
-	if (term1_validation.html.length > 0) {
-		html += term1_validation.html
-		return { type: 'html', html: html }
+	if (term1_validation.text.length > 0) {
+		text += term1_validation.text
+		return { type: 'text', text }
 	} else {
 		pp_plot_json.term = term1_validation.term_type
 		if (term1_validation.category == 'float' || term1_validation.category == 'integer') {
@@ -143,9 +143,9 @@ function validate_summary_response(response: string, common_genes: string[], dat
 
 	if (response_type.term2) {
 		const term2_validation = validate_term(response_type.term2, common_genes, dataset_json, ds)
-		if (term2_validation.html.length > 0) {
-			html += term2_validation.html
-			return { type: 'html', html: html }
+		if (term2_validation.text.length > 0) {
+			text += term2_validation.text
+			return { type: 'text', text }
 		} else {
 			pp_plot_json.term2 = term2_validation.term_type
 			if (term2_validation.category == 'float' || term2_validation.category == 'integer') {
@@ -164,8 +164,8 @@ function validate_summary_response(response: string, common_genes: string[], dat
 	const resolved = resolveChildType(pp_plot_json.category, pp_plot_json.category2, llmChildType)
 
 	if (resolved.error) {
-		html += resolved.error
-		return { type: 'html', html: html }
+		text += resolved.error
+		return { type: 'text', text }
 	} else {
 		pp_plot_json.childType = resolved.childType
 		// For two numeric variables displayed as violin/boxplot, discretize term2
@@ -185,9 +185,9 @@ function validate_summary_response(response: string, common_genes: string[], dat
 
 	if (response_type.simpleFilter && response_type.simpleFilter.length > 0) {
 		const validated_filters = validate_filter(response_type.simpleFilter, ds, '')
-		if (validated_filters.html.length > 0) {
-			html += validated_filters.html
-			return { type: 'html', html: html }
+		if (validated_filters.text.length > 0) {
+			text += validated_filters.text
+			return { type: 'text', text }
 		} else {
 			pp_plot_json.filter = validated_filters.simplefilter
 		}

server/routes/chat/utils.ts

@@ -161,7 +161,7 @@ export function extractGenesFromPrompt(prompt: string, genes_list: string[]): st
 }
 
 export function validate_term(response_term: string, common_genes: string[], dataset_json: any, ds: any) {
-	let html = ''
+	let text = ''
 	let term_type: any
 	let category: string = ''
 	const term: any =
@@ -172,21 +172,21 @@ export function validate_term(response_term: string, common_genes: string[], dat
 		const gene_hits = common_genes.filter(gene => gene == response_term.toLowerCase())
 		if (gene_hits.length == 0) {
 			// Neither a clinical term nor a gene
-			html += 'invalid term id:' + response_term
+			text += 'invalid term id:' + response_term
 		} else {
 			if (dataset_json.hasGeneExpression) {
 				// Check to see if dataset support gene expression
 				term_type = { term: { gene: response_term.toUpperCase(), type: 'geneExpression' } }
 				category = 'float'
 			} else {
-				html += 'Dataset does not support gene expression'
+				text += 'Dataset does not support gene expression'
 			}
 		}
 	} else {
 		term_type = { id: term.id }
 		category = term.type
 	}
-	return { term_type: term_type, html: html, category: category }
+	return { term_type, text, category }
 }
 
 export async function parse_geneset_db(genedb: string) {