Merge branch 'dev'

Author: thiswillbeyourgithub

DocToolsLLM/DocToolsLLM.py

@@ -78,7 +78,7 @@
 class DocToolsLLM_class:
     "This docstring is dynamically replaced by the content of DocToolsLLM/docs/USAGE.md"
 
-    VERSION: str = "0.38"
+    VERSION: str = "0.39"
 
     #@optional_typecheck
     @typechecked
@@ -113,7 +113,7 @@ def __init__(
         query_condense_question: Union[bool, int] = True,
 
         summary_n_recursion: int = 1,
-        summary_language: str = "[same as input]",
+        summary_language: str = "the same language as the document",
 
         llm_verbosity: Union[bool, int] = False,
         debug: Union[bool, int] = False,
@@ -427,7 +427,9 @@ def ntfy(text: str) -> str:
 
             if self.task == "summary_then_query":
                 whi("Done summarizing. Switching to query mode.")
-                if self.modelbackend == "openai":
+                if "logit_bias" in litellm.get_supported_openai_params(
+                        model=f"{self.modelbackend}/{self.modelname}",
+                    ):
                     del self.llm.model_kwargs["logit_bias"]
             else:
                 whi("Done summarizing. Exiting.")
@@ -473,7 +475,9 @@ def _summary_task(self) -> dict:
             else:
                 red(f"Cost estimate > limit but the api_base was modified so not crashing.")
 
-        if self.modelbackend == "openai":
+        if "logit_bias" in litellm.get_supported_openai_params(
+                model=f"{self.modelbackend}/{self.modelname}",
+            ):
             # increase likelyhood that chatgpt will use indentation by
             # biasing towards adding space.
             logit_val = 3
@@ -510,8 +514,17 @@ def _summary_task(self) -> dict:
                 56899: logit_val,    # "                                                                            "
                 98517: logit_val,    # "                                                                                "
                 }
+        if "frequency_penalty" in litellm.get_supported_openai_params(
+                model=f"{self.modelbackend}/{self.modelname}",
+            ):
             self.llm.model_kwargs["frequency_penalty"] = 0.0
+        if "presence_penalty" in litellm.get_supported_openai_params(
+                model=f"{self.modelbackend}/{self.modelname}",
+            ):
             self.llm.model_kwargs["presence_penalty"] = 0.0
+        if "temperature" in litellm.get_supported_openai_params(
+                model=f"{self.modelbackend}/{self.modelname}",
+            ):
             self.llm.model_kwargs["temperature"] = 0.0
 
         @optional_typecheck
@@ -717,7 +730,61 @@ def summarize_documents(
             red(f"Cost discrepancy? Tokens used according to the callback: '{llmcallback.total_tokens}' (${total_cost:.5f})")
         return results
 
-    def prepare_query_task(self):
+    @optional_typecheck
+    def prepare_query_task(self) -> None:
+        # set argument that are better suited for querying
+        if "logit_bias" in litellm.get_supported_openai_params(
+                model=f"{self.modelbackend}/{self.modelname}",
+            ):
+            # increase likelyhood that chatgpt will use indentation by
+            # biasing towards adding space.
+            logit_val = 3
+            self.llm.model_kwargs["logit_bias"] = {
+                12: logit_val,  # '-'
+                # 220: logit_val,  # ' '
+                # 532: logit_val,  # ' -'
+                # 9: logit_val,  # '*'
+                # 1635: logit_val,  # ' *'
+                # 197: logit_val,  # '\t'
+                334: logit_val,  # '**'
+                # 25: logit_val,  # ':'
+                # 551: logit_val,  # ' :'
+                # 13: -1,  # '.'
+                # logit bias for indentation, the number of space, because it consumes less token than using \t
+                257: logit_val,      # "    "
+                260: logit_val,      # "        "
+                1835: logit_val,     # "            "
+                338: logit_val,      # "                "
+                3909: logit_val,     # "                    "
+                5218: logit_val,     # "                        "
+                6663: logit_val,     # "                            "
+                792: logit_val,      # "                                "
+                10812: logit_val,    # "                                    "
+                13137: logit_val,    # "                                        "
+                15791: logit_val,    # "                                            "
+                19273: logit_val,    # "                                                "
+                25343: logit_val,    # "                                                    "
+                29902: logit_val,    # "                                                        "
+                39584: logit_val,    # "                                                            "
+                5341: logit_val,     # "                                                                "
+                52168: logit_val,    # "                                                                    "
+                38244: logit_val,    # "                                                                        "
+                56899: logit_val,    # "                                                                            "
+                98517: logit_val,    # "                                                                                "
+                }
+        if "frequency_penalty" in litellm.get_supported_openai_params(
+                model=f"{self.modelbackend}/{self.modelname}",
+            ):
+            self.llm.model_kwargs["frequency_penalty"] = 0.0
+        if "presence_penalty" in litellm.get_supported_openai_params(
+                model=f"{self.modelbackend}/{self.modelname}",
+            ):
+            self.llm.model_kwargs["presence_penalty"] = 0.0
+        if "temperature" in litellm.get_supported_openai_params(
+                model=f"{self.modelbackend}/{self.modelname}",
+            ):
+            self.llm.model_kwargs["temperature"] = 0.0
+
         # load embeddings for querying
         self.loaded_embeddings, self.embeddings = load_embeddings(
             embed_model=self.embed_model,

DocToolsLLM/docs/USAGE.md

@@ -150,7 +150,7 @@
     * If `--out_file` is used, each intermediate summary will be saved
     with the name `{out_file}.n.md` with n being the n-1th recursive summary.
 
-* `--summary_language`: str, default `"[same as input]"`
+* `--summary_language`: str, default `"the same language as the document"`
     * When writing a summary, the LLM will write using the language
     specified in this argument. If it's `[same as input]`, the LLM
     will not translate.

DocToolsLLM/utils/llm.py

@@ -17,7 +17,7 @@
 from langchain_core.outputs.llm_result import LLMResult
 from langchain_community.llms import FakeListLLM
 from langchain_community.chat_models import ChatLiteLLM
-from langchain_community.chat_models import ChatOpenAI
+from langchain_openai import ChatOpenAI
 from langchain_community.cache import SQLiteCache
 
 from .logger import whi, red, yel

DocToolsLLM/utils/prompts.py

@@ -7,26 +7,34 @@
 # PROMPT FOR SUMMARY TASKS
 BASE_SUMMARY_PROMPT = ChatPromptTemplate.from_messages(
         [
-        ("system", """You are Alfred, my best journalist. Your job today is to summarize in a specific way a text section I just sent you, but I'm not interested simply in high level takeaways. What I'm interested in is the thought process of the author(s), the reasonning, the arguments used etc. Your summary has to be as quick and easy to read as possible while following the rules.
-This is very important to me so if you succeed, I'll tip you up to $2000!
+        ("system", """You are Alfred, the best of my team. Your task today is to summarize in a specific way a text section I just sent you, but I'm not only interested in high level takeaways. I also need the thought process present in the document, the reasonning followed, the arguments used etc. But your summary has to be as quick and easy to read as possible while following specific instructions.
+This is very important to me so if you succeed, I'll pay you up to $2000 depending on how well you did!
 
-- Detailed instructions:
- ```
+Detailed instructions:
+```
+- Take a deep breath before answering
 - Include:
-    - All noteworthy information, anecdotes, facts, insights, definitions, clarifications, explanations, ideas, technical details, etc.
+    - All noteworthy information, anecdotes, facts, insights, definitions, clarifications, explanations, ideas, technical details, etc
+    - Epistemic indicators: you need to make explicit what markers of uncertainty for each information
 - Exclude:
-    - Sponsors, advertisements, etc.
-    - Jokes, ramblings.
-    - End of page references, tables of content, sources, links etc.
-    - When in doubt, keep the information in your summary.
+    - Sponsors, advertisements, etc
+    - Jokes, ramblings
+    - End of page references and sources, tables of content, links etc
+    - When in doubt about wether to include an information, include it
 - Format:
-    - Use markdown format: that means logical indentation, bullet points, bold etc. Don't use headers.
-    - Don't use complete sentence, I'm in a hurry and need bullet points.
-    - Use one bullet point per information, with the use of logical indentation this makes the whole piece quick and easy to skim.
-    - Use bold for important concepts (i.e. "- Mentions that **dietary supplements are healty** because ...")
-    - Write in {language}.
-    - Reformulate direct quotes to be concise, but stay faithful to the tone of the author.
-    - Avoid repetitions:  e.g. don't start several bullet points by 'The author thinks that', just say it once then use indentation to make it implied..
+    - Use markdown format: that means logical indentation, bullet points, bold etc
+        - Don't use headers
+        - Use bold for important concepts, and italic for epistemic markers
+            - ie "- *In his opinion*, **dietary supplements** are **healty** because ..."
+    - Stay faithful to the tone of the author
+    - You don't always have to use full sentences: you can ignore end of line punctuation etc
+        - BUT it is more important to be unambiguous and truthful than concise
+        - EVERY TIME POSSIBLE: use direct quote, 'formatted like that'
+    - Use one bullet point per information
+        - With the use of logical indentation this makes the whole piece quick and easy to skim
+    - Write your summary in {language}
+    - Avoid repetitions
+        - eg don't start several bullet points by 'The author thinks that', just say it once then use indented children bullet points to make it implicit
 ```"""),
         ("human", """{recursion_instruction}{metadata}{previous_summary}
 
@@ -37,7 +45,7 @@
         ],
 )
 # if the summary is recursive, add those instructions
-RECURSION_INSTRUCTION = "\nBut today, I'm giving you back your own summary because it was too long and contained repetition. I want you to rewrite it as closely as possible while removing repetitions and fixing the logical indentation. Of course you have to remve the 'Chunk' indicator if present, to curate the logical indentation. You can reorganize the text freely as long as you don't lose relevant information and follow the instructions I gave you. This is important."
+RECURSION_INSTRUCTION = "Actually, I'm giving you back your own summary from last time because it was too long and contained repetitions. I want you to rewrite it as closely as possible while removing repetitions and fixing the logical indentation. Of course you have to remove the 'Chunk' indicator if present, to curate the logical indentation. You can reorganize the text freely as long as you don't lose relevant information and follow the instructions I gave you before and right now. This is important."
 
 # PROMPT FOR QUERY TASKS
 PR_CONDENSE_QUESTION = ChatPromptTemplate.from_messages(
@@ -57,30 +65,57 @@
 
 PR_ANSWER_ONE_DOC = ChatPromptTemplate.from_messages(
     [
-        ("system", """You are an assistant for question-answering tasks.
-You are given a piece of document and a question to answer.
-If the document is ENTIRELY irrelevant to the question, answer directly 'IRRELEVANT' without anything else and no other formatting.
-Otherwise, use a maximum of 3 md bulletpoints to answer the question using only information from the provided document.
-Use markdown formatting for easier reading, but don't wrap your answer in a code block or anything like that: reply instantly without acknowledging those rules.
-Doing all that you have to remain VERY concise while remaining truthful to the document content.
-But DON'T interpret the question too strictly, e.g. the question can be implicit because phrased as an instruction like "give me all information about such and such", use common sense!"""),
+        ("system", """Given a piece of document and a question, your task is to answer the question while following specific instructions.
+
+Detailed instructions:
+```
+- Use markdown formatting
+    - Use bullet points, but no headers, bold, italic etc.
+    - Use logic based indentation for the bullet points.
+    - DON'T wrap your answer in a code block or anything like that.
+- Take a deep breath before answering.
+    - But then reply directly without acknowledging your task.
+- Use a maximum of 5 markdown bullet points to answer the question.
+    - If the document is ENTIRELY irrelevant to the question, answer simply 'IRRELEVANT' and NOTHING ELSE (especially no formatting).
+    - EVERY TIME POSSIBLE: use direct quote from the document, 'formatted like that'.
+    - DON'T use your own knowledge of the subject, only use the document.
+    - Remain as concise as possible, you can use [...] in your quotes to remove unecessary text.
+- DON'T interpret the question too strictly:
+    - eg: if the question is phrased as an instruction like "give me all information about such and such", use common sense and satisfy the instruction!
+```"""),
         ("human", "Question: '{question_to_answer}'\nContext:\n```\n{context}\n```\nWhat's your reply?")
     ]
 )
 
 PR_COMBINE_INTERMEDIATE_ANSWERS = ChatPromptTemplate.from_messages(
     [
-        ("system", """Given some statements and an answer, your task it to first answer directly the question in a md bullet point, then combine all additional information as additional bullet points. You must only use information from the statements.
-BUT, and above all: if the statements are not enough to answer the question you MUST start your answer by: 'OPINION:' followed by your answer using your own knowledge to let me know the source is you!
-No redundant bullet points must remain: you must combine redundant bullet points into a single more complicated bullet point.
+        ("system", """Given some statements and an answer, your task is to:
+1. answer directly the question using markdown bullet points
+2. then combine all additional information as additional bullet points.
 
-Ignore statements that are completely irrelevant to the question.
-Don't narrate, just do what I asked without acknowledging those rules.
-If the question contains acronyms, reuse them without specifying what they mean.
-Use markdown format, with bullet points and indentation etc.
-Be concise but don't omit ANY information from the statements.
-Answer in the same language as the question.
-But DON'T interpret the question too strictly, for example if the question makes reference to "documents" consider that it's what I call here "statements" for example. For example, if the question is rather an instruction like "give me all information about such and such", use common sense and don't be too strict!"""),
+Detailed instructions:
+```
+- Take a deep breath before answering.
+- Format:
+    - Use markdown format, with bullet points.
+      - IMPORTANT: use logical indentation to organize information hierarchically.
+      - The present instructions are a good example of proper formatting.
+    - Don't narrate, just do what I asked without acknowledging those rules.
+    - Reuse acronyms without specifying what they mean.
+    - Be concise but don't omit only irrelevant information from the statements.
+    - Answer in the same language as the question.
+- What to include:
+    - Only use information from the provided statements.
+        - IMPORTANT: if the statements are insufficient to answer the question you MUST start your answer by: 'OPINION:' followed by your own answer.
+            - This way I know the source is you!
+    - Ignore statements that are completely irrelevant to the question.
+    - Semi relevant statements can be included, especially if related to possible followup questions.
+    - No redundant information must remain.
+        - eg: fix redundancies with one parent bullet point and several indented children.
+    - DON'T interpret the question too strictly:
+        - eg: if the question makes reference to "documents" consider that it's what I call here "statements" for example.
+        - eg: if the question is phrased as an instruction like "give me all information about such and such", use common sense and satisfy the instruction!
+```"""),
         ("human", "Question: `{question}`\nStatements:\n```\n{intermediate_answers}\n```\nYour answer?""")
     ]
 )