Query engine with non-chatcompletion model

[d3buggerdan]

@dosu how can i use a query engine or a way to chat with my retriever with a non-chat completition model such as gpt 4. when i did it with 3.5 it worked but now it doesnt

[dosubot[bot]]

To use a query engine or chat with your retriever using a non-chat completion model such as GPT-4, follow these steps:

1. Install the necessary packages:

   %pip install llama-index-readers-file pymupdf
   %pip install llama-index-llms-openai
   %pip install llama-index-retrievers-bm25
   %pip install llama-index

2. Authenticate with OpenAI:

   import os
   
   os.environ["OPENAI_API_KEY"] = "sk-..."

3. Download and prepare your data:

   !mkdir data
   !wget --user-agent "Mozilla" "https://arxiv.org/pdf/2307.09288.pdf" -O "data/llama2.pdf"

4. Load documents:

   from pathlib import Path
   from llama_index.readers.file import PyMuPDFReader
   
   loader = PyMuPDFReader()
   documents = loader.load(file_path="./data/llama2.pdf")

5. Setup models:

   from llama_index.llms.openai import OpenAI
   from llama_index.embeddings.openai import OpenAIEmbedding
   
   llm = OpenAI(model="gpt-3.5-turbo", temperature=0.1)
   embed_model = OpenAIEmbedding(
       model="text-embedding-3-small", embed_batch_size=256
   )

6. Load into Vector Store:

   from llama_index.core import VectorStoreIndex
   from llama_index.core.node_parser import SentenceSplitter
   
   splitter = SentenceSplitter(chunk_size=1024)
   index = VectorStoreIndex.from_documents(
       documents, transformations=[splitter], embed_model=embed_model
   )

7. Define Advanced Retriever:

   from typing import List
   from llama_index.core import PromptTemplate
   from llama_index.core.schema import NodeWithScore
   from llama_index.core.retrievers import BaseRetriever
   import asyncio
   
   query_str = "How do the models developed in this work compare to open-source chat models based on the benchmarks tested?"
   
   query_gen_prompt_str = (
       "You are a helpful assistant that generates multiple search queries based on a "
       "single input query. Generate {num_queries} search queries, one on each line, "
       "related to the following input query:\n"
       "Query: {query}\n"
       "Queries:\n"
   )
   query_gen_prompt = PromptTemplate(query_gen_prompt_str)
   
   def generate_queries(llm, query_str: str, num_queries: int = 4):
       fmt_prompt = query_gen_prompt.format(
           num_queries=num_queries - 1, query=query_str
       )
       response = llm.complete(fmt_prompt)
       queries = response.text.split("\n")
       return queries
   
   queries = generate_queries(llm, query_str, num_queries=4)
   
   from tqdm.asyncio import tqdm
   
   async def run_queries(queries, retrievers):
       tasks = []
       for query in queries:
           for i, retriever in enumerate(retrievers):
               tasks.append(retriever.aretrieve(query))
   
       task_results = await tqdm.gather(*tasks)
   
       results_dict = {}
       for i, (query, query_result) in enumerate(zip(queries, task_results)):
           results_dict[(query, i)] = query_result
   
       return results_dict
   
   from llama_index.retrievers.bm25 import BM25Retriever
   
   vector_retriever = index.as_retriever(similarity_top_k=2)
   bm25_retriever = BM25Retriever.from_defaults(
       docstore=index.docstore, similarity_top_k=2
   )
   
   results_dict = await run_queries(queries, [vector_retriever, bm25_retriever])
   
   def fuse_results(results_dict, similarity_top_k: int = 2):
       k = 60.0
       fused_scores = {}
       text_to_node = {}
   
       for nodes_with_scores in results_dict.values():
           for rank, node_with_score in enumerate(
               sorted(
                   nodes_with_scores, key=lambda x: x.score or 0.0, reverse=True
               )
           ):
               text = node_with_score.node.get_content()
               text_to_node[text] = node_with_score
               if text not in fused_scores:
                   fused_scores[text] = 0.0
               fused_scores[text] += 1.0 / (rank + k)
   
       reranked_results = dict(
           sorted(fused_scores.items(), key=lambda x: x[1], reverse=True)
       )
   
       reranked_nodes: List[NodeWithScore] = []
       for text, score in reranked_results.items():
           reranked_nodes.append(text_to_node[text])
           reranked_nodes[-1].score = score
   
       return reranked_nodes[:similarity_top_k]
   
   final_results = fuse_results(results_dict)
   
   for n in final_results:
       print(n.score, "\n", n.text, "\n********\n")

8. Plug into RetrieverQueryEngine:

   from llama_index.core.query_engine import RetrieverQueryEngine
   
   class FusionRetriever(BaseRetriever):
       def __init__(
           self,
           llm,
           retrievers: List[BaseRetriever],
           similarity_top_k: int = 2,
       ) -> None:
           self._retrievers = retrievers
           self._similarity_top_k = similarity_top_k
           self._llm = llm
           super().__init__()
   
       def _retrieve(self, query_bundle: QueryBundle) -> List[NodeWithScore]:
           queries = generate_queries(
               self._llm, query_bundle.query_str, num_queries=4
           )
           results = asyncio.run(run_queries(queries, self._retrievers))
           final_results = fuse_results(
               results, similarity_top_k=self._similarity_top_k
           )
   
           return final_results
   
   fusion_retriever = FusionRetriever(
       llm, [vector_retriever, bm25_retriever], similarity_top_k=2
   )
   
   query_engine = RetrieverQueryEngine(fusion_retriever)
   
   response = query_engine.query(query_str)
   
   print(str(response))

By following these steps, you can set up a query engine to chat with your retriever using a non-chat completion model like GPT-4 ^[1][2][3][4].

To continue talking to Dosu, mention @dosu.

[dosubot[bot]]

To ensure that your custom templates are applied correctly and the required variables are provided, you need to make sure that the context variable is correctly passed to the templates. Here is a step-by-step guide to achieve this:

1. Define your custom prompt templates:

   from llama_index.core import PromptTemplate
   
   text_qa_template_str = (
       "Say hi at the start. Given the context provided, answer the following question with as much detail and relevant information as possible. "
       "Ensure to include explanations and any necessary background information to make the response informative and comprehensive.\n"
       "Context: {context}\n"
       "Question: {question}\n"
       "Answer:\n"
   )
   text_qa_template = PromptTemplate(text_qa_template_str)
   
   refine_template_str = (
       "Say hi at the start. Given the new context, refine the original answer to better answer the question. "
       "If the context isn't useful, output the original answer again. Ensure the refined answer is detailed and includes relevant explanations.\n"
       "Original Answer: {original_answer}\n"
       "New Context: {new_context}\n"
       "Refined Answer:\n"
   )
   refine_template = PromptTemplate(refine_template_str)

2. Create a BaseSynthesizer with your custom templates:

   from llama_index.core.response_synthesizers import BaseSynthesizer
   
   response_synthesizer = BaseSynthesizer(
       text_qa_template=text_qa_template,
       refine_template=refine_template
   )

3. Initialize the RetrieverQueryEngine with the custom response_synthesizer:

   from llama_index.core.query_engine.retriever_query_engine import RetrieverQueryEngine
   from llama_index.core.base.base_retriever import BaseRetriever
   
   # Assuming you have an instance of BaseRetriever
   retriever = BaseRetriever()  # Replace with your actual retriever
   
   # Initialize the RetrieverQueryEngine with the custom response synthesizer
   query_engine = RetrieverQueryEngine.from_args(
       retriever=retriever,
       response_synthesizer=response_synthesizer
   )

4. Ensure the required variables are provided during the query:

   query_str = "How does this contract handle the scenario of a tenant deciding not to renew the lease?"
   context_str = "Provide the relevant context here."  # Replace with actual context
   
   response = query_engine.query(query_str, context_str=context_str)
   print(response)

5. Update the prompts in the query_engine if needed:

   query_engine.update_prompts(
       {
           "response_synthesizer:text_qa_template": text_qa_template,
           "response_synthesizer:refine_template": refine_template
       }
   )

6. Verify the updated prompts:

   prompts_dict = query_engine.get_prompts()
   for k, p in prompts_dict.items():
       print(f"Prompt Key: {k}\nText:\n{p.get_template()}\n")

By following these steps, you ensure that your custom text_qa_template and refine_template are applied correctly to the RetrieverQueryEngine through the response_synthesizer. This should include the direct context that the retriever pulled and adhere to the custom instructions you provided in the templates.

If you still encounter issues, ensure that the context_str and other required variables are correctly passed to the query method and that the templates are correctly formatted to use these variables ^[1][2].

To continue talking to Dosu, mention @dosu.

[d3buggerdan]

@dous context_str isnt a thing?

[dosubot[bot]]

The context_str variable is indeed not a predefined variable in the LlamaIndex framework. Instead, you should use the correct variable names expected by the templates and the RetrieverQueryEngine. The correct variable names are context and query_str.

Here is how you can ensure that the correct variables are used and passed to the templates:

1. Define your custom prompt templates:

   from llama_index.core import PromptTemplate
   
   text_qa_template_str = (
       "Say hi at the start. Given the context provided, answer the following question with as much detail and relevant information as possible. "
       "Ensure to include explanations and any necessary background information to make the response informative and comprehensive.\n"
       "Context: {context}\n"
       "Question: {query_str}\n"
       "Answer:\n"
   )
   text_qa_template = PromptTemplate(text_qa_template_str)
   
   refine_template_str = (
       "Say hi at the start. Given the new context, refine the original answer to better answer the question. "
       "If the context isn't useful, output the original answer again. Ensure the refined answer is detailed and includes relevant explanations.\n"
       "Original Answer: {existing_answer}\n"
       "New Context: {new_context}\n"
       "Refined Answer:\n"
   )
   refine_template = PromptTemplate(refine_template_str)

2. Create a BaseSynthesizer with your custom templates:

   from llama_index.core.response_synthesizers import BaseSynthesizer
   
   response_synthesizer = BaseSynthesizer(
       text_qa_template=text_qa_template,
       refine_template=refine_template
   )

3. Initialize the RetrieverQueryEngine with the custom response_synthesizer:

   from llama_index.core.query_engine.retriever_query_engine import RetrieverQueryEngine
   from llama_index.core.base.base_retriever import BaseRetriever
   
   # Assuming you have an instance of BaseRetriever
   retriever = BaseRetriever()  # Replace with your actual retriever
   
   # Initialize the RetrieverQueryEngine with the custom response synthesizer
   query_engine = RetrieverQueryEngine.from_args(
       retriever=retriever,
       response_synthesizer=response_synthesizer
   )

4. Ensure the required variables are provided during the query:

   query_str = "How does this contract handle the scenario of a tenant deciding not to renew the lease?"
   context = "Provide the relevant context here."  # Replace with actual context
   
   response = query_engine.query(query_str, context=context)
   print(response)

5. Update the prompts in the query_engine if needed:

   query_engine.update_prompts(
       {
           "response_synthesizer:text_qa_template": text_qa_template,
           "response_synthesizer:refine_template": refine_template
       }
   )

6. Verify the updated prompts:

   prompts_dict = query_engine.get_prompts()
   for k, p in prompts_dict.items():
       print(f"Prompt Key: {k}\nText:\n{p.get_template()}\n")

By following these steps, you ensure that your custom text_qa_template and refine_template are applied correctly to the RetrieverQueryEngine through the response_synthesizer. This should include the direct context that the retriever pulled and adhere to the custom instructions you provided in the templates.

If you still encounter issues, ensure that the context and other required variables are correctly passed to the query method and that the templates are correctly formatted to use these variables ^[1][2].

To continue talking to Dosu, mention @dosu.

[d3buggerdan]

@dosu this is wrong i dont need context why am i getting this error

[dosubot[bot]]

The KeyError: 'context' error indicates that the template is expecting a context variable, but it is not being provided. If you don't need the context variable, you should remove it from your templates.

Here is how you can modify your templates and ensure they are correctly applied:

1. Define your custom prompt templates without the context variable:

   from llama_index.core import PromptTemplate
   
   text_qa_template_str = (
       "Say hi at the start. Answer the following question with as much detail and relevant information as possible. "
       "Ensure to include explanations and any necessary background information to make the response informative and comprehensive.\n"
       "Question: {query_str}\n"
       "Answer:\n"
   )
   text_qa_template = PromptTemplate(text_qa_template_str)
   
   refine_template_str = (
       "Say hi at the start. Refine the original answer to better answer the question. "
       "If the context isn't useful, output the original answer again. Ensure the refined answer is detailed and includes relevant explanations.\n"
       "Original Answer: {existing_answer}\n"
       "New Context: {new_context}\n"
       "Refined Answer:\n"
   )
   refine_template = PromptTemplate(refine_template_str)

2. Create a BaseSynthesizer with your custom templates:

   from llama_index.core.response_synthesizers import BaseSynthesizer
   
   response_synthesizer = BaseSynthesizer(
       text_qa_template=text_qa_template,
       refine_template=refine_template
   )

3. Initialize the RetrieverQueryEngine with the custom response_synthesizer:

   from llama_index.core.query_engine.retriever_query_engine import RetrieverQueryEngine
   from llama_index.core.base.base_retriever import BaseRetriever
   
   # Assuming you have an instance of BaseRetriever
   retriever = BaseRetriever()  # Replace with your actual retriever
   
   # Initialize the RetrieverQueryEngine with the custom response synthesizer
   query_engine = RetrieverQueryEngine.from_args(
       retriever=retriever,
       response_synthesizer=response_synthesizer
   )

4. Query using the updated query_engine:

   query_str = "How does this contract handle the scenario of a tenant deciding not to renew the lease?"
   
   response = query_engine.query(query_str)
   print(response)

5. Update the prompts in the query_engine if needed:

   query_engine.update_prompts(
       {
           "response_synthesizer:text_qa_template": text_qa_template,
           "response_synthesizer:refine_template": refine_template
       }
   )

6. Verify the updated prompts:

   prompts_dict = query_engine.get_prompts()
   for k, p in prompts_dict.items():
       print(f"Prompt Key: {k}\nText:\n{p.get_template()}\n")

By following these steps, you ensure that your custom text_qa_template and refine_template are applied correctly to the RetrieverQueryEngine through the response_synthesizer. This should prevent the KeyError: 'context' error since the context variable is no longer required in the templates ^[1][2][3].

To continue talking to Dosu, mention @dosu.