Fix CI linting (#128)

* fix lint

* add exit hotfix for llm-tests target

---------

Signed-off-by: Jack Luar <[email protected]>

Author: luarss

Makefile

@@ -14,7 +14,9 @@ format:
 
 .PHONY: check
 check:
-	@for folder in $(FOLDERS); do (cd $$folder && make check && cd ../); done
+	@for folder in $(FOLDERS); do \
+ 	   (cd $$folder && make check && cd ../) || exit 1; \
+		done
 	@. ./backend/.venv/bin/activate && \
 		pre-commit run --all-files
 

backend/src/agents/retriever_graph.py

@@ -121,7 +121,7 @@ def agent(self, state: AgentState) -> dict[str, list[str]]:
             return {"tools": []}
 
         if self.inbuilt_tool_calling:
-            model = self.llm.bind_tools(self.tools, tool_choice="any")
+            model = self.llm.bind_tools(self.tools, tool_choice="any")  # type: ignore
 
             tool_choice_chain = (
                 ChatPromptTemplate.from_template(rephrase_prompt_template)

backend/src/chains/hybrid_retriever_chain.py

@@ -84,9 +84,11 @@ def create_hybrid_retriever(self) -> None:
             if path_flag and database_name in os.listdir(path):
                 if database_name in os.listdir(path):
                     similarity_retriever_chain.create_vector_db()
-                    similarity_retriever_chain.vector_db.load_db(database_name)
+                    similarity_retriever_chain.vector_db.load_db(database_name)  # type: ignore
                     self.vector_db = similarity_retriever_chain.vector_db
-                    self.vector_db.processed_docs = similarity_retriever_chain.vector_db.get_documents()
+                    self.vector_db.processed_docs = (  # type: ignore
+                        similarity_retriever_chain.vector_db.get_documents()  # type: ignore
+                    )
             else:
                 similarity_retriever_chain.embed_docs(return_docs=True)
                 self.vector_db = similarity_retriever_chain.vector_db

backend/src/chains/similarity_retriever_chain.py

@@ -13,6 +13,7 @@
 
 class SimilarityRetrieverChain(BaseChain):
     count = 0
+
     def __init__(
         self,
         llm_model: Optional[
@@ -64,38 +65,44 @@ def embed_docs(
         Optional[list[Document]],
         Optional[list[Document]],
     ]:
+        # Create the vector database if it does not exist
         if self.vector_db is None and extend_existing is False:
             self.create_vector_db()
 
-        if self.markdown_docs_path is not None and self.vector_db is not None:
+        assert (
+            self.vector_db is not None
+        ), "Vector DB must be created before embedding documents."
+        if self.markdown_docs_path is not None:
             self.processed_docs = self.vector_db.add_md_docs(
                 folder_paths=self.markdown_docs_path,
                 chunk_size=self.chunk_size,
                 return_docs=return_docs,
             )
 
-        if self.manpages_path is not None and self.vector_db is not None:
+        if self.manpages_path is not None:
             self.processed_manpages = self.vector_db.add_md_manpages(
                 folder_paths=self.manpages_path, return_docs=return_docs
             )
 
-        if self.other_docs_path is not None and self.vector_db is not None:
-            for folder_name in self.other_docs_path:
-                for root, _, files in os.walk(folder_name):
-                    for file in files:
-                        other_docs_path = os.path.join(root, file)
-                        if other_docs_path.endswith(".pdf"):
-                            self.processed_pdfs = self.vector_db.add_documents(
-                                file_paths=[other_docs_path],
-                                file_type="pdf",
-                                return_docs=return_docs,
-                            )
-                        else:
-                            raise ValueError(
-                                "File type not supported. Only PDFs are supported."
-                            )
-
-        if self.html_docs_path is not None and self.vector_db is not None:
+        if self.other_docs_path is not None:
+            pdf_files = [
+                os.path.join(root, file)
+                for folder_name in self.other_docs_path
+                for root, _, files in os.walk(folder_name)
+                for file in files
+                if file.endswith(".pdf")
+            ]
+
+            if not pdf_files:
+                raise ValueError("File type not supported. Only PDFs are supported.")
+
+            self.processed_pdfs = self.vector_db.add_documents(
+                file_paths=pdf_files,
+                file_type="pdf",
+                return_docs=return_docs,
+            )
+
+        if self.html_docs_path is not None:
             self.processed_html = self.vector_db.add_html(
                 folder_paths=self.html_docs_path,
                 return_docs=return_docs,

backend/src/vectorstores/faiss.py

@@ -205,10 +205,12 @@ def save_db(self, name) -> None:
 
     def load_db(self, name) -> None:
         load_path = f"{self.get_db_path()}/{name}"
-        self._faiss_db = FAISS.load_local(load_path, self.embedding_model, allow_dangerous_deserialization=True)
+        self._faiss_db = FAISS.load_local(
+            load_path, self.embedding_model, allow_dangerous_deserialization=True
+        )
 
     def get_documents(self) -> list[Document]:
-        return self._faiss_db.docstore._dict.values()
+        return self._faiss_db.docstore._dict.values()  # type: ignore
 
     def process_json(self, folder_paths: list[str]) -> FAISS:
         logging.info("Processing json files...")

evaluation/Makefile

@@ -30,4 +30,5 @@ clean:
 llm-tests: clean
 	@. .venv/bin/activate && \
 		cd auto_evaluation && \
-		./llm_tests.sh 2>&1 | tee llm_tests_output.txt
+		./llm_tests.sh 2>&1 | tee llm_tests_output.txt; \
+		exit $${PIPESTATUS[0]}

evaluation/auto_evaluation/src/models/vertex_ai.py

@@ -5,8 +5,8 @@
 
 import instructor
 
-from typing import Any
-from vertexai.generative_models import GenerativeModel, HarmBlockThreshold, HarmCategory  # type: ignore
+from typing import Any, Type
+from vertexai.generative_models import GenerativeModel, HarmBlockThreshold, HarmCategory
 from deepeval.models.base_model import DeepEvalBaseLLM
 from pydantic import BaseModel
 
@@ -31,18 +31,20 @@ def load_model(self, *args, **kwargs):
             HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE,
             HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE,
         }
+        if not self.model_name:
+            raise ValueError("Model name must be specified for Google Vertex AI.")
 
         return GenerativeModel(
             model_name=self.model_name,
             safety_settings=safety_settings,
         )
 
-    def generate(self, prompt: str, schema: BaseModel) -> Any:
+    def generate(self, prompt: str, schema: Type[BaseModel]) -> Any:
         instructor_client = instructor.from_vertexai(
             client=self.load_model(),
             mode=instructor.Mode.VERTEXAI_TOOLS,
         )
-        resp = instructor_client.messages.create(  # type: ignore
+        resp = instructor_client.messages.create(
             messages=[
                 {
                     "role": "user",
@@ -53,13 +55,12 @@ def generate(self, prompt: str, schema: BaseModel) -> Any:
         )
         return resp
 
-    async def a_generate(self, prompt: str, schema: BaseModel) -> Any:
+    async def a_generate(self, prompt: str, schema: Any) -> Any:
         instructor_client = instructor.from_vertexai(
             client=self.load_model(),
             mode=instructor.Mode.VERTEXAI_TOOLS,
-            _async=True,
         )
-        resp = await instructor_client.messages.create(  # type: ignore
+        resp = await instructor_client.completions.create(
             messages=[
                 {
                     "role": "user",
@@ -71,7 +72,7 @@ async def a_generate(self, prompt: str, schema: BaseModel) -> Any:
         return resp
 
     def get_model_name(self):
-        return self.model_name
+        return self.model_name or "model-not-specified"
 
 
 def main():
@@ -86,7 +87,7 @@ async def main_async():
     model = GoogleVertexAILangChain(model_name="gemini-1.5-pro-002")
     prompt = "Write me a joke"
     print(f"Prompt: {prompt}")
-    response = await model.a_generate(prompt, Response)
+    response = await model.a_generate(prompt, schema=Response)
     print(f"Response: {response}")
 
 

evaluation/human_evaluation/utils/sheets.py

@@ -106,10 +106,10 @@ def write_responses(responses: list[str], row_numbers: list[int]) -> int:
         result = (
             service.spreadsheets()
             .values()
-            .batchUpdate(spreadsheetId=SHEET_ID, body=body)  # type: ignore
+            .batchUpdate(spreadsheetId=SHEET_ID, body=body)
             .execute()
         )
-        return result.get("totalUpdatedCells")  # type: ignore
+        return result.get("totalUpdatedCells")
     except HttpError as error:
         st.error("Failed to write responses to the Google Sheet.")
         st.error(f"An error occurred: {error}")

evaluation/human_evaluation/utils/utils.py

@@ -176,7 +176,7 @@ def update_gform(questions_descriptions: list[dict[str, str]]) -> None:
         form_body = {"requests": requests}
         forms_service.forms().batchUpdate(
             formId=GOOGLE_FORM_ID,
-            body=form_body,  # type: ignore
+            body=form_body,
         ).execute()
 
         st.success("Google Form updated successfully.")

evaluation/init_google.py

@@ -1,3 +1,4 @@
+# type: ignore
 import os
 import argparse
 from google.oauth2.service_account import Credentials
@@ -47,7 +48,7 @@ def create_google_form(form_title: str, user_email: str) -> str:
     - str: Google Form ID.
     """
     form_metadata = {"info": {"title": form_title}}
-    form = forms_service.forms().create(body=form_metadata).execute()  # type: ignore
+    form = forms_service.forms().create(body=form_metadata).execute()
     form_id = form["formId"]
     print(f"Created Form with ID: {form_id}")
 
@@ -68,13 +69,13 @@ def create_google_sheet(sheet_title: str, user_email: str) -> str:
     - str: Google Sheet ID.
     """
     sheet_metadata = {"properties": {"title": sheet_title}}
-    sheet = sheets_service.spreadsheets().create(body=sheet_metadata).execute()  # type: ignore
+    sheet = sheets_service.spreadsheets().create(body=sheet_metadata).execute()
     sheet_id = sheet["spreadsheetId"]
     print(f"Created Sheet with ID: {sheet_id}")
 
     gc = gspread.authorize(creds)
-    sheet = gc.open_by_key(sheet_id).get_worksheet(0)  # type: ignore
-    sheet.append_row(["Questions", "Generated Answers"])  # type: ignore
+    sheet = gc.open_by_key(sheet_id).get_worksheet(0)
+    sheet.append_row(["Questions", "Generated Answers"])
 
     share_file(sheet_id, user_email)