[python/knowpro] Various improvements to tools (#1426)

* 595de6d vizcmp.py: Get scores from Score: headers instead of file
trailer
* fc5dcb6 utools.py: Always show output diff even when success differs
* 8147b7f utool.py: assign score 0.001 if we have an unexpected answer
* fa4053a utool.py: Add support for Pydantic's Logfire
* fca0f57 vizcmp.py: Simplify footer
* e29246e vizcmp.py: Sort by average score, not first file's
* f0e7ebf vizcmp.py: Only sort filenames when using default glob
* d2a6be4 vizcmp.py: Print only basename
* 6b564a2 vizcmp.py: show file names in footer
* 4e72780 vizcmp.py: Display N/A results in bright yellow

Author: gvanrossum-ms

python/ta/tools/utool.py

@@ -48,6 +48,33 @@
 from typeagent.podcasts import podcast
 
 
+### Logfire setup ###
+
+
+def setup_logfire():
+    import logfire
+
+    def scrubbing_callback(m: logfire.ScrubMatch):
+        # if m.path == ('attributes', 'http.request.header.authorization'):
+        #     return m.value
+
+        # if m.path == ('attributes', 'http.request.header.api-key'):
+        #     return m.value
+
+        if (
+            m.path == ("attributes", "http.request.body.text", "messages", 0, "content")
+            and m.pattern_match.group(0) == "secret"
+        ):
+            return m.value
+
+        # if m.path == ('attributes', 'http.response.header.azureml-model-session'):
+        #     return m.value
+
+    logfire.configure(scrubbing=logfire.ScrubbingOptions(callback=scrubbing_callback))
+    logfire.instrument_pydantic_ai()
+    logfire.instrument_httpx(capture_all=True)
+
+
 ### Classes ###
 
 
@@ -116,6 +143,8 @@ def main():
     parser = make_arg_parser("TypeAgent Query Tool")
     args = parser.parse_args()
     fill_in_debug_defaults(parser, args)
+    if args.logfire:
+        setup_logfire()
     settings = importing.ConversationSettings()
     query_context = load_podcast_index(args.podcast, settings)
     ar_list, ar_index = load_index_file(args.qafile, "question", QuestionAnswerData)
@@ -488,6 +517,11 @@ def make_arg_parser(description: str) -> argparse.ArgumentParser:
         action="store_true",
         help="Show the TypeScript schema computed by typechat.",
     )
+    debug.add_argument(
+        "--logfire",
+        action="store_true",
+        help="Upload log events to Pydantic's Logfire server",
+    )
 
     return parser
 
@@ -744,14 +778,16 @@ async def compare_answers(
     actual_text, actual_success = actual
 
     if expected_success != actual_success:
-        print(f"Expected success: {expected_success}; actual: {actual_success}")
-        return 0.000
+        print(
+            f"Expected success: {Fore.RED}{expected_success}{Fore.RESET}; "
+            f"actual: {Fore.GREEN}{actual_success}{Fore.RESET}"
+        )
 
-    if not actual_success:
+    elif not actual_success:
         print(Fore.GREEN + f"Both failed" + Fore.RESET)
         return 1.001
 
-    if expected_text == actual_text:
+    elif expected_text == actual_text:
         print(Fore.GREEN + f"Both equal" + Fore.RESET)
         return 1.000
 
@@ -760,7 +796,11 @@ async def compare_answers(
     else:
         n = 2
     print_diff(expected_text, actual_text, n=n)
-    return await equality_score(context, expected_text, actual_text)
+
+    if expected_success != actual_success:
+        return 0.000 if expected_success else 0.001  # 0.001 == Answer not expected
+    else:
+        return await equality_score(context, expected_text, actual_text)
 
 
 def print_diff(a: str, b: str, n: int) -> None:

python/ta/tools/vizcmp.py

@@ -11,7 +11,7 @@
 
 
 def main():
-    files = sys.argv[1:] or glob.glob("evals/eval-*.txt")
+    files = sys.argv[1:] or sorted(glob.glob("evals/eval-*.txt"))
     table = {}  # {file: {counter: score, ...}, ...}
     questions = {}  # {counter: question, ...}
 
@@ -20,56 +20,41 @@ def main():
         with open(file, "r") as f:
             lines = f.readlines()
 
+        scores = {}
         counter = None
         for i, line in enumerate(lines):
             if m := re.match(r"^(?:-+|\*+)\s+(\d+)\s+", line):
                 counter = int(m.group(1))
-            elif m := re.match(r"^.*; Question:\s+(.*)$", line):
-                question = m.group(1)
+            elif m := re.match(r"^Score:\s+([\d.]+); Question:\s+(.*)$", line):
+                score = float(m.group(1))
+                scores[counter] = score
+                question = m.group(2)
                 if counter not in questions:
                     questions[counter] = question
                 elif questions[counter] != question:
                     print(f"File {file} has a different question for {counter}:")
                     print(f"< {questions[counter]}")
                     print(f"> {question}")
 
-        i = lines.index("==================================================\n")
-        if i < 0:
-            print(f"File {file} does not contain a separator line")
-            continue
-        lines = lines[i + 1 :]
-        text = "".join(lines)
-        matches = re.findall(r"\d\.\d\d\d\(\d+\)", text)
-        if not matches:
-            print(f"File {file} does not contain any scores")
-            continue
-        # print(len(matches), matches)
-        data = {}
-        for match in matches:
-            m = re.match(r"(\d\.\d\d\d)\((\d+)\)", match)
-            assert m
-            score = float(m.group(1))
-            counter = int(m.group(2))
-            data[counter] = score
-        assert len(data) == len(matches)
-        table[file] = data
+        table[file] = scores
 
     # Print header
-    all_files = sorted(table.keys())
+    all_files = list(table.keys())
     print_header(all_files)
 
     # Print data
     all_counters = sorted(
         {counter for data in table.values() for counter in data.keys()},
-        key=lambda x: table[all_files[0]].get(x, 0.0),
+        key=lambda x: statistics.mean(table[file].get(x, 0.0) for file in all_files),
         reverse=True,
     )
     for counter in all_counters:
         print(f"{counter:>3}:", end="")
         for file in all_files:
             score = table[file].get(counter, None)
             if score is None:
-                output = "  N/A "
+                output = Fore.YELLOW + "  N/A " + Fore.RESET
+                output = Style.BRIGHT + output + Style.RESET_ALL
             else:
                 output = f"{score:.3f}"
                 output = f"{output:>6}"
@@ -87,7 +72,7 @@ def main():
         print(f" {questions.get(counter)}")
 
     # Print header again
-    print_header(all_files)
+    print_footer(all_files)
 
 
 def print_header(all_files):
@@ -103,5 +88,10 @@ def print_header(all_files):
     print()
 
 
+def print_footer(all_files):
+    for i, file in reversed(list(enumerate(all_files))):
+        print("     |" * i + "     " + os.path.basename(file))
+
+
 if __name__ == "__main__":
     main()