Remove token-counting library for conversation history truncation (#2449)

* Simplify RenderedPrompt class

* Fix reference

* Remove (EXAMPLE) from prompts

* Remove long history test

* Fix test with missing async, run on Windows too

* Remove Windows again as a few tests fail

Author: pamelafox

.github/workflows/python-test.yaml

@@ -59,7 +59,7 @@ jobs:
           run: black . --check --verbose
         - name: Run Python tests
           if: runner.os != 'Windows'
-          run: pytest -s -vv --cov --cov-fail-under=86
+          run: pytest -s -vv --cov --cov-fail-under=89
         - name: Run E2E tests with Playwright
           id: e2e
           if: runner.os != 'Windows'

app/backend/approaches/approach.py

@@ -92,10 +92,6 @@ class ThoughtStep:
 
 class Approach(ABC):
 
-    # Allows usage of non-GPT model even if no tokenizer is available for accurate token counting
-    # Useful for using local small language models, for example
-    ALLOW_NON_GPT_MODELS = True
-
     def __init__(
         self,
         search_client: SearchClient,
@@ -147,8 +143,8 @@ async def search(
         use_vector_search: bool,
         use_semantic_ranker: bool,
         use_semantic_captions: bool,
-        minimum_search_score: Optional[float],
-        minimum_reranker_score: Optional[float],
+        minimum_search_score: Optional[float] = None,
+        minimum_reranker_score: Optional[float] = None,
         use_query_rewriting: Optional[bool] = None,
     ) -> List[Document]:
         search_text = query_text if use_text_search else ""

app/backend/approaches/chatreadretrieveread.py

@@ -9,7 +9,6 @@
     ChatCompletionMessageParam,
     ChatCompletionToolParam,
 )
-from openai_messages_token_helper import build_messages, get_token_limit
 
 from approaches.approach import ThoughtStep
 from approaches.chatapproach import ChatApproach
@@ -53,7 +52,6 @@ def __init__(
         self.content_field = content_field
         self.query_language = query_language
         self.query_speller = query_speller
-        self.chatgpt_token_limit = get_token_limit(chatgpt_model, default_to_minimum=self.ALLOW_NON_GPT_MODELS)
         self.prompt_manager = prompt_manager
         self.query_rewrite_prompt = self.prompt_manager.load_prompt("chat_query_rewrite.prompty")
         self.query_rewrite_tools = self.prompt_manager.load_tools("chat_query_rewrite_tools.json")
@@ -99,30 +97,18 @@ async def run_until_final_call(
         if not isinstance(original_user_query, str):
             raise ValueError("The most recent message content must be a string.")
 
-        rendered_query_prompt = self.prompt_manager.render_prompt(
+        query_messages = self.prompt_manager.render_prompt(
             self.query_rewrite_prompt, {"user_query": original_user_query, "past_messages": messages[:-1]}
         )
         tools: List[ChatCompletionToolParam] = self.query_rewrite_tools
 
         # STEP 1: Generate an optimized keyword search query based on the chat history and the last question
-        query_response_token_limit = 100
-        query_messages = build_messages(
-            model=self.chatgpt_model,
-            system_prompt=rendered_query_prompt.system_content,
-            few_shots=rendered_query_prompt.few_shot_messages,
-            past_messages=rendered_query_prompt.past_messages,
-            new_user_content=rendered_query_prompt.new_user_content,
-            tools=tools,
-            max_tokens=self.chatgpt_token_limit - query_response_token_limit,
-            fallback_to_default=self.ALLOW_NON_GPT_MODELS,
-        )
-
         chat_completion: ChatCompletion = await self.openai_client.chat.completions.create(
             messages=query_messages,  # type: ignore
             # Azure OpenAI takes the deployment name as the model name
             model=self.chatgpt_deployment if self.chatgpt_deployment else self.chatgpt_model,
             temperature=0.0,  # Minimize creativity for search query generation
-            max_tokens=query_response_token_limit,  # Setting too low risks malformed JSON, setting too high may affect performance
+            max_tokens=100,  # Setting too low risks malformed JSON, setting too high may affect performance
             n=1,
             tools=tools,
             seed=seed,
@@ -153,7 +139,7 @@ async def run_until_final_call(
 
         # STEP 3: Generate a contextual and content specific answer using the search results and chat history
         text_sources = self.get_sources_content(results, use_semantic_captions, use_image_citation=False)
-        rendered_answer_prompt = self.prompt_manager.render_prompt(
+        messages = self.prompt_manager.render_prompt(
             self.answer_prompt,
             self.get_system_prompt_variables(overrides.get("prompt_template"))
             | {
@@ -164,16 +150,6 @@ async def run_until_final_call(
             },
         )
 
-        response_token_limit = 1024
-        messages = build_messages(
-            model=self.chatgpt_model,
-            system_prompt=rendered_answer_prompt.system_content,
-            past_messages=rendered_answer_prompt.past_messages,
-            new_user_content=rendered_answer_prompt.new_user_content,
-            max_tokens=self.chatgpt_token_limit - response_token_limit,
-            fallback_to_default=self.ALLOW_NON_GPT_MODELS,
-        )
-
         extra_info = {
             "data_points": {"text": text_sources},
             "thoughts": [
@@ -220,7 +196,7 @@ async def run_until_final_call(
             model=self.chatgpt_deployment if self.chatgpt_deployment else self.chatgpt_model,
             messages=messages,
             temperature=overrides.get("temperature", 0.3),
-            max_tokens=response_token_limit,
+            max_tokens=1024,
             n=1,
             stream=should_stream,
             seed=seed,

app/backend/approaches/chatreadretrievereadvision.py

@@ -9,7 +9,6 @@
     ChatCompletionMessageParam,
     ChatCompletionToolParam,
 )
-from openai_messages_token_helper import build_messages, get_token_limit
 
 from approaches.approach import ThoughtStep
 from approaches.chatapproach import ChatApproach
@@ -64,7 +63,6 @@ def __init__(
         self.query_speller = query_speller
         self.vision_endpoint = vision_endpoint
         self.vision_token_provider = vision_token_provider
-        self.chatgpt_token_limit = get_token_limit(gpt4v_model, default_to_minimum=self.ALLOW_NON_GPT_MODELS)
         self.prompt_manager = prompt_manager
         self.query_rewrite_prompt = self.prompt_manager.load_prompt("chat_query_rewrite.prompty")
         self.query_rewrite_tools = self.prompt_manager.load_tools("chat_query_rewrite_tools.json")
@@ -97,30 +95,18 @@ async def run_until_final_call(
             raise ValueError("The most recent message content must be a string.")
 
         # Use prompty to prepare the query prompt
-        rendered_query_prompt = self.prompt_manager.render_prompt(
+        query_messages = self.prompt_manager.render_prompt(
             self.query_rewrite_prompt, {"user_query": original_user_query, "past_messages": messages[:-1]}
         )
         tools: List[ChatCompletionToolParam] = self.query_rewrite_tools
 
         # STEP 1: Generate an optimized keyword search query based on the chat history and the last question
-        query_response_token_limit = 100
-        query_model = self.chatgpt_model
-        query_deployment = self.chatgpt_deployment
-        query_messages = build_messages(
-            model=query_model,
-            system_prompt=rendered_query_prompt.system_content,
-            few_shots=rendered_query_prompt.few_shot_messages,
-            past_messages=rendered_query_prompt.past_messages,
-            new_user_content=rendered_query_prompt.new_user_content,
-            max_tokens=self.chatgpt_token_limit - query_response_token_limit,
-        )
-
         chat_completion: ChatCompletion = await self.openai_client.chat.completions.create(
             messages=query_messages,
             # Azure OpenAI takes the deployment name as the model name
-            model=query_deployment if query_deployment else query_model,
+            model=self.chatgpt_deployment if self.chatgpt_deployment else self.chatgpt_model,
             temperature=0.0,  # Minimize creativity for search query generation
-            max_tokens=query_response_token_limit,
+            max_tokens=100,
             n=1,
             tools=tools,
             seed=seed,
@@ -166,7 +152,7 @@ async def run_until_final_call(
                 if url:
                     image_sources.append(url)
 
-        rendered_answer_prompt = self.prompt_manager.render_prompt(
+        messages = self.prompt_manager.render_prompt(
             self.answer_prompt,
             self.get_system_prompt_variables(overrides.get("prompt_template"))
             | {
@@ -178,16 +164,6 @@ async def run_until_final_call(
             },
         )
 
-        response_token_limit = 1024
-        messages = build_messages(
-            model=self.gpt4v_model,
-            system_prompt=rendered_answer_prompt.system_content,
-            past_messages=rendered_answer_prompt.past_messages,
-            new_user_content=rendered_answer_prompt.new_user_content,
-            max_tokens=self.chatgpt_token_limit - response_token_limit,
-            fallback_to_default=self.ALLOW_NON_GPT_MODELS,
-        )
-
         extra_info = {
             "data_points": {
                 "text": text_sources,
@@ -198,9 +174,9 @@ async def run_until_final_call(
                     "Prompt to generate search query",
                     query_messages,
                     (
-                        {"model": query_model, "deployment": query_deployment}
-                        if query_deployment
-                        else {"model": query_model}
+                        {"model": self.chatgpt_model, "deployment": self.chatgpt_deployment}
+                        if self.chatgpt_deployment
+                        else {"model": self.chatgpt_model}
                     ),
                 ),
                 ThoughtStep(
@@ -236,7 +212,7 @@ async def run_until_final_call(
             model=self.gpt4v_deployment if self.gpt4v_deployment else self.gpt4v_model,
             messages=messages,
             temperature=overrides.get("temperature", 0.3),
-            max_tokens=response_token_limit,
+            max_tokens=1024,
             n=1,
             stream=should_stream,
             seed=seed,

app/backend/approaches/promptmanager.py

@@ -1,20 +1,10 @@
 import json
 import pathlib
-from dataclasses import dataclass
 
 import prompty
 from openai.types.chat import ChatCompletionMessageParam
 
 
-@dataclass
-class RenderedPrompt:
-    all_messages: list[ChatCompletionMessageParam]
-    system_content: str
-    few_shot_messages: list[ChatCompletionMessageParam]
-    past_messages: list[ChatCompletionMessageParam]
-    new_user_content: str
-
-
 class PromptManager:
 
     def load_prompt(self, path: str):
@@ -23,7 +13,7 @@ def load_prompt(self, path: str):
     def load_tools(self, path: str):
         raise NotImplementedError
 
-    def render_prompt(self, prompt, data) -> RenderedPrompt:
+    def render_prompt(self, prompt, data) -> list[ChatCompletionMessageParam]:
         raise NotImplementedError
 
 
@@ -37,40 +27,5 @@ def load_prompt(self, path: str):
     def load_tools(self, path: str):
         return json.loads(open(self.PROMPTS_DIRECTORY / path).read())
 
-    def render_prompt(self, prompt, data) -> RenderedPrompt:
-        # Assumes that the first message is the system message, the last message is the user message,
-        # and the messages in-between are either examples or past messages.
-
-        all_messages: list = prompty.prepare(prompt, data)
-        remaining_messages = all_messages.copy()
-
-        system_content = None
-        if all_messages[0]["role"] == "system":
-            system_content = all_messages[0]["content"]
-            remaining_messages.pop(0)
-        else:
-            raise ValueError("The first message in the prompt must be a system message.")
-
-        new_user_content = None
-        if all_messages[-1]["role"] == "user":
-            new_user_content = all_messages[-1]["content"]
-            remaining_messages.pop(-1)
-        else:
-            raise ValueError("The last message in the prompt must be a user message.")
-
-        few_shot_messages = []
-        past_messages = []
-        for user_message, assistant_message in zip(remaining_messages[0::2], remaining_messages[1::2]):
-            if user_message["content"].startswith("(EXAMPLE)"):
-                user_message["content"] = user_message["content"][9:].lstrip()
-                few_shot_messages.extend([user_message, assistant_message])
-            else:
-                past_messages.extend([user_message, assistant_message])
-
-        return RenderedPrompt(
-            all_messages=all_messages,
-            system_content=system_content,
-            few_shot_messages=few_shot_messages,
-            past_messages=past_messages,
-            new_user_content=new_user_content,
-        )
+    def render_prompt(self, prompt, data) -> list[ChatCompletionMessageParam]:
+        return prompty.prepare(prompt, data)

app/backend/approaches/prompts/ask_answer_question.prompty

@@ -23,7 +23,7 @@ If you cannot answer using the sources below, say you don't know. Use below exam
 {% endif %}
 
 user:
-(EXAMPLE) What is the deductible for the employee plan for a visit to Overlake in Bellevue?
+What is the deductible for the employee plan for a visit to Overlake in Bellevue?
 
 Sources:
 info1.txt: deductibles depend on whether you are in-network or out-of-network. In-network deductibles are $500 for employee and $1000 for family. Out-of-network deductibles are $1000 for employee and $2000 for family.

app/backend/approaches/prompts/chat_query_rewrite.prompty

@@ -24,13 +24,13 @@ If the question is not in English, translate the question to English before gene
 If you cannot generate a search query, return just the number 0.
 
 user:
-(EXAMPLE) How did crypto do last year?
+How did crypto do last year?
 
 assistant:
 Summarize Cryptocurrency Market Dynamics from last year
 
 user:
-(EXAMPLE) What are my health plans?
+What are my health plans?
 
 assistant:
 Show available health plans

app/backend/approaches/retrievethenread.py

@@ -4,7 +4,6 @@
 from azure.search.documents.models import VectorQuery
 from openai import AsyncOpenAI
 from openai.types.chat import ChatCompletionMessageParam
-from openai_messages_token_helper import get_token_limit
 
 from approaches.approach import Approach, ThoughtStep
 from approaches.promptmanager import PromptManager
@@ -48,7 +47,6 @@ def __init__(
         self.content_field = content_field
         self.query_language = query_language
         self.query_speller = query_speller
-        self.chatgpt_token_limit = get_token_limit(chatgpt_model, self.ALLOW_NON_GPT_MODELS)
         self.prompt_manager = prompt_manager
         self.answer_prompt = self.prompt_manager.load_prompt("ask_answer_question.prompty")
 
@@ -95,7 +93,7 @@ async def run(
 
         # Process results
         text_sources = self.get_sources_content(results, use_semantic_captions, use_image_citation=False)
-        rendered_answer_prompt = self.prompt_manager.render_prompt(
+        messages = self.prompt_manager.render_prompt(
             self.answer_prompt,
             self.get_system_prompt_variables(overrides.get("prompt_template"))
             | {"user_query": q, "text_sources": text_sources},
@@ -104,7 +102,7 @@ async def run(
         chat_completion = await self.openai_client.chat.completions.create(
             # Azure OpenAI takes the deployment name as the model name
             model=self.chatgpt_deployment if self.chatgpt_deployment else self.chatgpt_model,
-            messages=rendered_answer_prompt.all_messages,
+            messages=messages,
             temperature=overrides.get("temperature", 0.3),
             max_tokens=1024,
             n=1,
@@ -133,7 +131,7 @@ async def run(
                 ),
                 ThoughtStep(
                     "Prompt to generate answer",
-                    rendered_answer_prompt.all_messages,
+                    messages,
                     (
                         {"model": self.chatgpt_model, "deployment": self.chatgpt_deployment}
                         if self.chatgpt_deployment

app/backend/approaches/retrievethenreadvision.py

@@ -6,7 +6,6 @@
 from openai.types.chat import (
     ChatCompletionMessageParam,
 )
-from openai_messages_token_helper import get_token_limit
 
 from approaches.approach import Approach, ThoughtStep
 from approaches.promptmanager import PromptManager
@@ -56,7 +55,6 @@ def __init__(
         self.query_speller = query_speller
         self.vision_endpoint = vision_endpoint
         self.vision_token_provider = vision_token_provider
-        self.gpt4v_token_limit = get_token_limit(gpt4v_model, self.ALLOW_NON_GPT_MODELS)
         self.prompt_manager = prompt_manager
         self.answer_prompt = self.prompt_manager.load_prompt("ask_answer_question_vision.prompty")
 
@@ -123,15 +121,15 @@ async def run(
                 if url:
                     image_sources.append(url)
 
-        rendered_answer_prompt = self.prompt_manager.render_prompt(
+        messages = self.prompt_manager.render_prompt(
             self.answer_prompt,
             self.get_system_prompt_variables(overrides.get("prompt_template"))
             | {"user_query": q, "text_sources": text_sources, "image_sources": image_sources},
         )
 
         chat_completion = await self.openai_client.chat.completions.create(
             model=self.gpt4v_deployment if self.gpt4v_deployment else self.gpt4v_model,
-            messages=rendered_answer_prompt.all_messages,
+            messages=messages,
             temperature=overrides.get("temperature", 0.3),
             max_tokens=1024,
             n=1,
@@ -161,7 +159,7 @@ async def run(
                 ),
                 ThoughtStep(
                     "Prompt to generate answer",
-                    rendered_answer_prompt.all_messages,
+                    messages,
                     (
                         {"model": self.gpt4v_model, "deployment": self.gpt4v_deployment}
                         if self.gpt4v_deployment