Add multimodal JSONL ground truth

Author: pamelafox

app/backend/approaches/approach.py

@@ -364,7 +364,8 @@ def nonewlines(s: str) -> str:
         for doc in results:
             # Get the citation for the source page
             citation = self.get_citation(doc.sourcepage)
-            citations.append(citation)
+            if citation not in citations:
+                citations.append(citation)
 
             # If semantic captions are used, extract captions; otherwise, use content
             if use_semantic_captions and doc.captions:

app/backend/approaches/prompts/ask_answer_question.prompty

@@ -14,26 +14,20 @@ system:
 {% if override_prompt %}
 {{ override_prompt }}
 {% else %}
-You are an intelligent assistant helping Contoso Inc employees with their healthcare plan questions and employee handbook questions.
-Use 'you' to refer to the individual asking the questions even if they ask with 'I'.
-Answer the following question using only the data provided in the sources below.
-Each source has a name followed by colon and the actual information, always include the source name for each fact you use in the response.
-If you cannot answer using the sources below, say you don't know. Use below example to answer.
+Assistant helps the company employees with their questions about internal documents. Be brief in your answers.
+Answer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below.
+You CANNOT ask clarifying questions to the user, since the user will have no way to reply.
+If the question is not in English, answer in the language used in the question.
+Each source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].
 {% if image_sources %}
 Each image source has the document file name in the top left corner of the image with coordinates (10,10) pixels with format <filename.ext#page=N>,
 and the image figure name is right-aligned in the top right corner of the image.
 The filename of the actual image is in the top right corner of the image and is in the format <figureN_N.png>.
 Each text source starts in a new line and has the file name followed by colon and the actual information.
 Always include the source document filename for each fact you use in the response in the format: [document_name.ext#page=N].
 If you are referencing an image, add the image filename in the format: [document_name.ext#page=N(image_name.png)].
-Answer the following question using only the data provided in the sources below.
-If you cannot answer using the sources below, say you don't know.
-Return just the answer without any input texts.
 {% endif %}
-Possible citations for current question:
-{% for citation in citations %}
-[{{ citation }}]
-{% endfor %}
+Possible citations for current question: {% for citation in citations %} [{{ citation }}] {% endfor %}
 {{ injected_prompt }}
 {% endif %}
 

app/backend/approaches/prompts/chat_answer_question.prompty

@@ -20,22 +20,20 @@ system:
 {% if override_prompt %}
 {{ override_prompt }}
 {% else %}
-Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.
-Answer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.
+Assistant helps the company employees with their questions about internal documents. Be brief in your answers.
+Answer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below.
+If asking a clarifying question to the user would help, ask the question.
 If the question is not in English, answer in the language used in the question.
 Each source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].
-{% if include_images %}
+{% if image_sources %}
 Each image source has the document file name in the top left corner of the image with coordinates (10,10) pixels with format <filename.ext#page=N>,
 and the image figure name is right-aligned in the top right corner of the image.
 The filename of the actual image is in the top right corner of the image and is in the format <figureN_N.png>.
 Each text source starts in a new line and has the file name followed by colon and the actual information
-Always include the source name from the image or text for each fact you use in the response in the format: [filename]
-Answer the following question using only the data provided in the sources below.
-If asking a clarifying question to the user would help, ask the question.
-Be brief in your answers.
-The text and image source can be the same file name, don't use the image title when citing the image source, only use the file name as mentioned
-If you cannot answer using the sources below, say you don't know. Return just the answer without any input texts.
+Always include the source document filename for each fact you use in the response in the format: [document_name.ext#page=N].
+If you are referencing an image, add the image filename in the format: [document_name.ext#page=N(image_name.png)].
 {% endif %}
+Possible citations for current question: {% for citation in citations %} [{{ citation }}] {% endfor %}
 {{ injected_prompt }}
 {% endif %}
 
@@ -56,9 +54,9 @@ Make sure the last question ends with ">>".
 
 user:
 {{ user_query }}
-{% for image_source in image_sources %}
+{% if image_sources is defined %}{% for image_source in image_sources %}
 ![Image]({{image_source}})
-{% endfor %}
+{% endfor %}{% endif %}
 {% if text_sources is defined %}
 Sources:
 {% for text_source in text_sources %}

evals/ground_truth_multimodal.jsonl

@@ -0,0 +1,31 @@
+{"question": "How closely do the S&P 500 and NASDAQ move together?",
+ "truth": "The S&P 500 and NASDAQ move very closely together, with a correlation coefficient of 0.95, indicating a strong positive relationship between the two indices [Financial Market Analysis Report 2023.pdf#page=7]."
+}
+{"question": "Which commodity—oil, gold, or wheat—was the most stable over the last decade?",
+ "truth": "Over the last decade, gold was the most stable commodity compared to oil and wheat. The annual percentage changes for gold mostly stayed within a smaller range, while oil showed significant fluctuations including a large negative change in 2014 and a large positive peak in 2021. Wheat also varied but less than oil and more than gold [Financial Market Analysis Report 2023.pdf#page=6][Financial Market Analysis Report 2023.pdf#page=6(figure6_1.png)]."
+}
+{"question": "Do cryptocurrencies like Bitcoin or Ethereum show stronger ties to stocks or commodities?",
+"truth": "Cryptocurrencies like Bitcoin and Ethereum show stronger ties to stocks than to commodities. The correlation values between Bitcoin and stock indices are 0.3 with the S&P 500 and 0.4 with NASDAQ, while for Ethereum, the correlations are 0.35 with the S&P 500 and 0.45 with NASDAQ. In contrast, the correlations with commodities like Oil are lower (0.2 for Bitcoin and 0.25 for Ethereum), and correlations with Gold are slightly negative (-0.1 for Bitcoin and -0.05 for Ethereum) [Financial Market Analysis Report 2023.pdf#page=7]."
+}
+{"question": "Around what level did the S&P 500 reach its highest point before declining in 2021?",
+"truth": "The S&P 500 reached its highest point just above the 4500 level before declining in 2021 [Financial Market Analysis Report 2023.pdf#page=4][Financial Market Analysis Report 2023.pdf#page=4(figure4_1.png)]."
+}
+{"question": "In which month of 2023 did Bitcoin nearly hit 45,000?",
+ "truth": "Bitcoin nearly hit 45,000 in December 2023, as shown by the blue line reaching close to 45,000 on the graph for that month [Financial Market Analysis Report 2023.pdf#page=5(figure5_1.png)]."
+}
+{
+ "question": "Which year saw oil prices fall the most, and by roughly how much did they drop?",
+  "truth": "The year that saw oil prices fall the most was 2020, with a drop of roughly 20% as shown by the blue bar extending to about -20% on the horizontal bar chart of annual percentage changes for Oil from 2014 to 2022 [Financial Market Analysis Report 2023.pdf#page=6(figure6_1.png)]."
+}
+{"question": "What was the approximate inflation rate in 2022?",
+ "truth": "The approximate inflation rate in 2022 was near 3.4% according to the orange line in the inflation data on the graph showing trends from 2018 to 2023 [Financial Market Analysis Report 2023.pdf#page=8(figure8_1.png)]."
+}
+{"question": "By 2028, to what relative value are oil prices projected to move compared to their 2024 baseline of 100?",
+"truth" :"Oil prices are projected to decline to about 90 by 2028, relative to their 2024 baseline of 100. [Financial Market Analysis Report 2023.pdf#page=9(figure9_1.png)]."
+}
+{"question": "What approximate value did the S&P 500 fall to at its lowest point between 2018 and 2022?",
+ "truth": "The S&P 500 fell in 2018 to an approximate value of around 2600 at its lowest point between 2018 and 2022, as shown by the graph depicting the 5-Year Trend of the S&P 500 Index [Financial Market Analysis Report 2023.pdf#page=4(figure4_1.png)]."
+}
+{"question": "Around what value did Ethereum finish the year at in 2023?",
+"truth": "Ethereum finished the year 2023 at a value around 2200, as indicated by the orange line on the price fluctuations graph for the last 12 months [Financial Market Analysis Report 2023.pdf#page=5][Financial Market Analysis Report 2023.pdf#page=5(figure5_1.png)][Financial Market Analysis Report 2023.pdf#page=5(figure5_2.png)]."
+}

scripts/pretty_print_jsonl.py

@@ -0,0 +1,128 @@
+"""Utility to pretty-format a JSONL (JSON Lines) file.
+
+NOTE: Classic JSONL expects one JSON object per single line. Once we pretty
+print (indent) each object, the result is no longer *strict* JSONL because
+objects will span multiple lines. This script offers a few output modes so
+you can choose what you need:
+
+1. Default (stdout): Pretty prints each record (with indentation) separated
+   by a blank line for readability.
+2. --in-place: Rewrites the source file by replacing each original single-line
+   object with its multi-line, indented representation separated by a blank line.
+3. --output <path>: Writes the pretty output to a new file (recommended if you
+   also want to keep the original valid JSONL file unchanged).
+4. --as-array: Instead of individual objects, emit a single JSON array containing
+   all objects, using indentation (this produces standard JSON, not JSONL).
+
+Examples:
+  python scripts/pretty_print_jsonl.py evals/ground_truth_multimodal.jsonl
+  python scripts/pretty_print_jsonl.py evals/ground_truth_multimodal.jsonl --output evals/ground_truth_multimodal.pretty.jsonl
+  python scripts/pretty_print_jsonl.py evals/ground_truth_multimodal.jsonl --in-place
+  python scripts/pretty_print_jsonl.py evals/ground_truth_multimodal.jsonl --as-array --output evals/ground_truth_multimodal.pretty.json
+
+Safeguards:
+  * Refuses to use --in-place together with --as-array (ambiguous expectations).
+  * Backs up the original file to <filename>.bak before in-place rewrite unless
+    --no-backup is supplied.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+
+def read_jsonl(path: Path):
+    """Yield parsed JSON objects from a JSONL file.
+
+    Skips empty lines. Raises ValueError with context on parse failures.
+    """
+    for idx, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
+        stripped = line.strip()
+        if not stripped:
+            continue
+        try:
+            yield json.loads(stripped)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Failed to parse JSON on line {idx} of {path}: {e}") from e
+
+
+def write_pretty_individual(objs, indent: int) -> str:
+    """Return a string with each object pretty JSON, separated by a blank line."""
+    parts = [json.dumps(o, indent=indent, ensure_ascii=False) for o in objs]
+    # Add trailing newline for file friendliness
+    return "\n\n".join(parts) + "\n"
+
+
+def write_pretty_array(objs, indent: int) -> str:
+    return json.dumps(list(objs), indent=indent, ensure_ascii=False) + "\n"
+
+
+def parse_args(argv: list[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Pretty-format a JSONL file.")
+    parser.add_argument(
+        "jsonl_file",
+        type=Path,
+        help="Path to the source JSONL file (one JSON object per line).",
+    )
+    parser.add_argument("--indent", type=int, default=2, help="Indent level for json.dumps (default: 2)")
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        "--in-place",
+        action="store_true",
+        help="Rewrite the original file with pretty-formatted objects (not strict JSONL).",
+    )
+    group.add_argument(
+        "--output",
+        type=Path,
+        help="Path to write output. If omitted and not --in-place, prints to stdout.",
+    )
+    parser.add_argument(
+        "--as-array",
+        action="store_true",
+        help="Emit a single JSON array instead of individual pretty objects.",
+    )
+    parser.add_argument(
+        "--no-backup",
+        action="store_true",
+        help="When using --in-place, do not create a .bak backup file.",
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv or sys.argv[1:])
+
+    if not args.jsonl_file.exists():
+        print(f"Error: File not found: {args.jsonl_file}", file=sys.stderr)
+        return 1
+
+    objs = list(read_jsonl(args.jsonl_file))
+
+    if args.as_array:
+        output_text = write_pretty_array(objs, args.indent)
+    else:
+        output_text = write_pretty_individual(objs, args.indent)
+
+    # Destination logic
+    if args.in_place:
+        if not args.no_backup:
+            backup_path = args.jsonl_file.with_suffix(args.jsonl_file.suffix + ".bak")
+            if not backup_path.exists():
+                backup_path.write_text(args.jsonl_file.read_text(encoding="utf-8"), encoding="utf-8")
+        args.jsonl_file.write_text(output_text, encoding="utf-8")
+        print(f"Rewrote {args.jsonl_file} ({len(objs)} objects).")
+    elif args.output:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(output_text, encoding="utf-8")
+        print(f"Wrote pretty output to {args.output} ({len(objs)} objects).")
+    else:
+        # stdout
+        sys.stdout.write(output_text)
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

tests/snapshots/test_app/test_ask_prompt_template_concat/client0/result.json

@@ -54,7 +54,7 @@
             {
                 "description": [
                     {
-                        "content": "You are an intelligent assistant helping Contoso Inc employees with their healthcare plan questions and employee handbook questions.\nUse 'you' to refer to the individual asking the questions even if they ask with 'I'.\nAnswer the following question using only the data provided in the sources below.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response.\nIf you cannot answer using the sources below, say you don't know. Use below example to answer.\n\nPossible citations for current question:\n\n[Benefit_Options-2.pdf]\n\n Meow like a cat.",
+                        "content": "You are an intelligent assistant helping Contoso Inc employees with their healthcare plan questions and employee handbook questions.\nUse 'you' to refer to the individual asking the questions even if they ask with 'I'.\nAnswer the following question using only the data provided in the sources below.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\nIf you cannot answer using the sources below, say you don't know. Use below example to answer.\n\nPossible citations for current question:\n\n[Benefit_Options-2.pdf]\n\n Meow like a cat.",
                         "role": "system"
                     },
                     {

tests/snapshots/test_app/test_ask_prompt_template_concat/client1/result.json

@@ -54,7 +54,7 @@
             {
                 "description": [
                     {
-                        "content": "You are an intelligent assistant helping Contoso Inc employees with their healthcare plan questions and employee handbook questions.\nUse 'you' to refer to the individual asking the questions even if they ask with 'I'.\nAnswer the following question using only the data provided in the sources below.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response.\nIf you cannot answer using the sources below, say you don't know. Use below example to answer.\n\nPossible citations for current question:\n\n[Benefit_Options-2.pdf]\n\n Meow like a cat.",
+                        "content": "You are an intelligent assistant helping Contoso Inc employees with their healthcare plan questions and employee handbook questions.\nUse 'you' to refer to the individual asking the questions even if they ask with 'I'.\nAnswer the following question using only the data provided in the sources below.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\nIf you cannot answer using the sources below, say you don't know. Use below example to answer.\n\nPossible citations for current question:\n\n[Benefit_Options-2.pdf]\n\n Meow like a cat.",
                         "role": "system"
                     },
                     {

tests/snapshots/test_app/test_ask_rtr_hybrid/client0/result.json

@@ -54,7 +54,7 @@
             {
                 "description": [
                     {
-                        "content": "You are an intelligent assistant helping Contoso Inc employees with their healthcare plan questions and employee handbook questions.\nUse 'you' to refer to the individual asking the questions even if they ask with 'I'.\nAnswer the following question using only the data provided in the sources below.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response.\nIf you cannot answer using the sources below, say you don't know. Use below example to answer.\n\nPossible citations for current question:\n\n[Benefit_Options-2.pdf]",
+                        "content": "Assistant helps the company employees with their questions about internal documents. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below.\nYou CANNOT ask clarifying questions to the user, since the user will have no way to reply.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\n\nPossible citations for current question:  [Benefit_Options-2.pdf]",
                         "role": "system"
                     },
                     {