Make llm-complete-guide work again (#164)

* fix url scraper

* update requirements

* fix outdated code

* Update ZenML model version and fix vector store metadata access

* Upgrade ZenML requirement to version 0.73.0

* Change default index type to Postgres in index generator

* update constants

* Add log_metadata import from ZenML in evaluation step

* Suppress FutureWarning and refactor logging in eval and index steps

* formatting

* run evals in parallel

* Add tenacity to requirements for improved retry handling

* run tests in parallel

* Add LLM-judged evaluation function for RAG tests

Introduces a new function `run_llm_judged_tests` to perform end-to-end tests on RAG systems using LLM evaluation. The implementation includes:
- Parallel processing of test cases
- Scoring for toxicity, faithfulness, helpfulness, and relevance
- Retry logic for robust test execution
- Detailed logging of test results

* Add metadata logging for comprehensive evaluation metrics

Enhance the evaluation visualization step by logging detailed metrics to ZenML, including:
- Retrieval performance metrics
- Generation failure rates
- Quality scores (toxicity, faithfulness, helpfulness, relevance)
- Composite scores for overall quality and retrieval effectiveness

* Clean up imports and remove unused imports in evaluation steps

Refactor import statements in eval_retrieval.py and eval_visualisation.py to:
- Remove unused imports
- Organize imports consistently
- Simplify import statements

* Remove commented section in RAG configuration file

Simplify the dev/rag.yaml configuration by removing the commented "environment configuration" line, keeping the configuration clean and concise.

* Adjust default temperature for OpenAI model completion

Modify the default temperature parameter in get_completion_from_messages() from 0.4 to 0, ensuring more deterministic and focused model responses.

* make query via CLI work again

* Update deployment command in README for simplified RAG pipeline deployment

* Add type safety for ZenML secrets in Hugging Face deployment

Modify Hugging Face space deployment to ensure ZenML store secrets are converted to strings before adding, preventing potential type-related errors during deployment.

* Add Elasticsearch and Tenacity to project requirements

Update project dependencies to include:
- Elasticsearch for potential search and indexing functionality
- Tenacity for improved retry handling in various components

* Update ZenML chatbot model constants and improve vector store retrieval

- Add explicit constants for ZenML chatbot model name and version
- Enhance find_vectorstore_name() function with error handling and fallback mechanism
- Improve logging for vector store metadata retrieval

* Fix deployment :)

Author: strickvl

llm-complete-guide/README.md

@@ -100,7 +100,7 @@ use for the LLM.
 When you're ready to make the query, run the following command:
 
 ```shell
-python run.py query "how do I use a custom materializer inside my own zenml steps? i.e. how do I set it? inside the @step decorator?" --model=gpt4
+python run.py query --query-text "how do I use a custom materializer inside my own zenml steps? i.e. how do I set it? inside the @step decorator?" --model=gpt4
 ```
 
 Alternative options for LLMs to use include:
@@ -147,13 +147,7 @@ export ZENML_HF_SPACE_NAME=<YOUR_HF_SPACE_NAME> # optional, defaults to "llm-com
 To deploy the RAG pipeline, you can use the following command:
 
 ```shell
-python run.py --deploy
-```
-
-Alternatively, you can run the basic RAG pipeline *and* deploy it in one go:
-
-```shell
-python run.py --rag --deploy
+python run.py deploy
 ```
 
 This will open a Hugging Face space in your browser where you can interact with

llm-complete-guide/configs/dev/rag.yaml

@@ -1,6 +1,5 @@
 enable_cache: False
 
-# environment configuration
 settings:
   docker:
     requirements:

llm-complete-guide/constants.py

@@ -17,14 +17,16 @@
 import os
 
 # Vector Store constants
-CHUNK_SIZE = 2000
+CHUNK_SIZE = 1000
 CHUNK_OVERLAP = 50
 EMBEDDING_DIMENSIONALITY = (
     384  # Update this to match the dimensionality of the new model
 )
 
 # ZenML constants
 ZENML_CHATBOT_MODEL = "zenml-docs-qa-chatbot"
+ZENML_CHATBOT_MODEL_NAME = "zenml-docs-qa-chatbot"
+ZENML_CHATBOT_MODEL_VERSION = "0.71.0-dev"
 
 # Scraping constants
 RATE_LIMIT = 5  # Maximum number of requests per second
@@ -35,8 +37,8 @@
 MODEL_NAME_MAP = {
     "gpt4": "gpt-4",
     "gpt35": "gpt-3.5-turbo",
-    "claude3": "claude-3-opus-20240229",
-    "claudehaiku": "claude-3-haiku-20240307",
+    "claude3": "claude-3-5-sonnet-latest",
+    "claudehaiku": "claude-3-5-haiku-latest",
 }
 
 # CHUNKING_METHOD = "split-by-document"

llm-complete-guide/deployment_hf.py

@@ -1,13 +1,44 @@
+import logging
+
 import gradio as gr
+from constants import SECRET_NAME
 from utils.llm_utils import process_input_with_retrieval
+from zenml.client import Client
 
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
 
-def predict(message, history):
-    return process_input_with_retrieval(
-        input=message,
-        n_items_retrieved=20,
-        use_reranking=True,
+# Initialize ZenML client and verify secret access
+try:
+    client = Client()
+    secret = client.get_secret(SECRET_NAME)
+    logger.info(
+        f"Successfully initialized ZenML client and found secret {SECRET_NAME}"
     )
+except Exception as e:
+    logger.error(f"Failed to initialize ZenML client or access secret: {e}")
+    raise RuntimeError(f"Application startup failed: {e}")
+
+
+def predict(message, history):
+    try:
+        return process_input_with_retrieval(
+            input=message,
+            n_items_retrieved=20,
+            use_reranking=True,
+        )
+    except Exception as e:
+        logger.error(f"Error processing message: {e}")
+        return f"Sorry, I encountered an error: {str(e)}"
+
 
+# Launch the Gradio interface
+interface = gr.ChatInterface(
+    predict,
+    title="ZenML Documentation Assistant",
+    description="Ask me anything about ZenML!",
+)
 
-gr.ChatInterface(predict, type="messages").launch()
+if __name__ == "__main__":
+    interface.launch(server_name="0.0.0.0", share=False)

llm-complete-guide/gh_action_rag.py

@@ -21,12 +21,10 @@
 
 import click
 import yaml
-from zenml.enums import PluginSubType
-
 from pipelines.llm_index_and_evaluate import llm_index_and_evaluate
-from zenml.client import Client
 from zenml import Model
-from zenml.exceptions import ZenKeyError
+from zenml.client import Client
+from zenml.enums import PluginSubType
 
 
 @click.command(
@@ -89,7 +87,7 @@ def main(
     zenml_model_name: Optional[str] = "zenml-docs-qa-rag",
     zenml_model_version: Optional[str] = None,
 ):
-    """ 
+    """
     Executes the pipeline to train a basic RAG model.
 
     Args:
@@ -108,14 +106,14 @@ def main(
         config = yaml.safe_load(file)
 
     # Read the model version from a file in the root of the repo
-    #  called "ZENML_VERSION.txt". 
+    #  called "ZENML_VERSION.txt".
     if zenml_model_version == "staging":
         postfix = "-rc0"
     elif zenml_model_version == "production":
         postfix = ""
     else:
         postfix = "-dev"
-  
+
     if Path("ZENML_VERSION.txt").exists():
         with open("ZENML_VERSION.txt", "r") as file:
             zenml_model_version = file.read().strip()
@@ -177,7 +175,7 @@ def main(
                 service_account_id=service_account_id,
                 auth_window=0,
                 flavor="builtin",
-                action_type=PluginSubType.PIPELINE_RUN
+                action_type=PluginSubType.PIPELINE_RUN,
             ).id
             client.create_trigger(
                 name="Production Trigger LLM-Complete",

llm-complete-guide/pipelines/__init__.py

@@ -19,5 +19,5 @@
 from pipelines.generate_chunk_questions import generate_chunk_questions
 from pipelines.llm_basic_rag import llm_basic_rag
 from pipelines.llm_eval import llm_eval
+from pipelines.llm_index_and_evaluate import llm_index_and_evaluate
 from pipelines.rag_deployment import rag_deployment
-from pipelines.llm_index_and_evaluate import llm_index_and_evaluate

llm-complete-guide/pipelines/finetune_embeddings.py

@@ -12,7 +12,6 @@
 #  or implied. See the License for the specific language governing
 #  permissions and limitations under the License.
 
-from constants import EMBEDDINGS_MODEL_NAME_ZENML
 from steps.finetune_embeddings import (
     evaluate_base_model,
     evaluate_finetuned_model,

llm-complete-guide/pipelines/llm_basic_rag.py

@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from litellm import config_path
 
 from steps.populate_index import (
     generate_embeddings,

llm-complete-guide/pipelines/llm_index_and_evaluate.py

@@ -15,9 +15,10 @@
 # limitations under the License.
 #
 
-from pipelines import llm_basic_rag, llm_eval
 from zenml import pipeline
 
+from pipelines import llm_basic_rag, llm_eval
+
 
 @pipeline
 def llm_index_and_evaluate() -> None:

llm-complete-guide/requirements.txt

@@ -1,4 +1,4 @@
-zenml[server]
+zenml[server]>=0.73.0
 ratelimit
 pgvector
 psycopg2-binary
@@ -21,6 +21,7 @@ torch
 gradio
 huggingface-hub
 elasticsearch
+tenacity
 
 # optional requirements for S3 artifact store
 # s3fs>2022.3.0