Feature/update complete llm (#140)

* Removed new-line

* Fixed github action

* Fixed workflow

* Fixed github action

* Ignore server mismatch

* Ignore server mismatch

* Added all requirements

* Fixed gh action

* More fine grained control over configs

* Added defaults

* fixed config path

* Automate trigger update

Author: AlexejPenner

.github/workflows/production_run_complete_llm.yml

@@ -0,0 +1,58 @@
+name: Production LLM-COMPLETE
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'llm-complete-guide/**'
+concurrency:
+  # New commit on branch cancels running workflows of the same branch
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-staging-workflow:
+    runs-on: ubuntu-latest
+    env:
+      ZENML_HOST: ${{ secrets.ZENML_HOST }}
+      ZENML_API_KEY: ${{ secrets.ZENML_API_KEY }}
+      ZENML_PRODUCTION_STACK : 51a49786-b82a-4646-bde7-a460efb0a9c5
+      ZENML_GITHUB_SHA: ${{ github.event.pull_request.head.sha }}
+      ZENML_GITHUB_URL_PR: ${{ github.event.pull_request._links.html.href }}
+      ZENML_DEBUG: true
+      ZENML_ANALYTICS_OPT_IN: false
+      ZENML_LOGGING_VERBOSITY: INFO
+      ZENML_PROJECT_SECRET_NAME: llm-complete
+      ZENML_DISABLE_CLIENT_SERVER_MISMATCH_WARNING: True
+      ZENML_ACTION_ID: 23a4d58c-bd2b-47d5-a41d-0a845d2982f8
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install requirements
+        working-directory: ./llm-complete-guide
+        run: |
+          pip3 install -r requirements.txt
+          pip3 install -r requirements-argilla.txt
+          zenml integration install gcp -y
+
+      - name: Connect to ZenML server
+        working-directory: ./llm-complete-guide
+        run: |
+          zenml init
+          zenml connect --url $ZENML_HOST --api-key $ZENML_API_KEY
+
+      - name: Set stack (Production)
+        working-directory: ./llm-complete-guide
+        run: |
+          zenml stack set ${{ env.ZENML_PRODUCTION_STACK }}
+
+      - name: Run pipeline, create pipeline, configure trigger (Production)
+        working-directory: ./llm-complete-guide
+        run: |
+          python scripts/gh_action_rag.py --no-cache --create-template --action-id  ${{ env.ZENML_ACTION_ID }}

.github/workflows/run_complete_llm.yml renamed to .github/workflows/staging_run_complete_llm.yml

@@ -10,6 +10,7 @@ concurrency:
 
 jobs:
   run-staging-workflow:
+    runs-on: ubuntu-latest
     env:
       ZENML_HOST: ${{ secrets.ZENML_HOST }}
       ZENML_API_KEY: ${{ secrets.ZENML_API_KEY }}
@@ -20,6 +21,7 @@ jobs:
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_LOGGING_VERBOSITY: INFO
       ZENML_PROJECT_SECRET_NAME: llm-complete
+      ZENML_DISABLE_CLIENT_SERVER_MISMATCH_WARNING: True
 
     steps:
       - name: Check out repository code
@@ -30,20 +32,24 @@ jobs:
           python-version: '3.11'
 
       - name: Install requirements
+        working-directory: ./llm-complete-guide
         run: |
           pip3 install -r requirements.txt
+          pip3 install -r requirements-argilla.txt
           zenml integration install gcp -y
 
       - name: Connect to ZenML server
+        working-directory: ./llm-complete-guide
         run: |
+          zenml init
           zenml connect --url $ZENML_HOST --api-key $ZENML_API_KEY
 
       - name: Set stack (Staging)
-        if: ${{ github.base_ref == 'staging' }}
+        working-directory: ./llm-complete-guide
         run: |
           zenml stack set ${{ env.ZENML_STAGING_STACK }}
 
       - name: Run pipeline (Staging)
-        if: ${{ github.base_ref == 'staging' }}
+        working-directory: ./llm-complete-guide
         run: |
-          python run.py --rag --evaluation --no-cache
+          python scripts/gh_action_rag.py --no-cache

llm-complete-guide/configs/rag_gcp.yaml

@@ -11,6 +11,7 @@ settings:
       - psycopg2-binary
       - tiktoken
       - ratelimit
+      - rerankers
     environment:
       ZENML_PROJECT_SECRET_NAME: llm_complete
       ZENML_ENABLE_RICH_TRACEBACK: FALSE
@@ -20,13 +21,13 @@ steps:
   url_scraper:
     parameters:
       docs_url: https://docs.zenml.io
-  generate_embeddings:
-    step_operator: "terraform-gcp-6c0fd52233ca"
-    settings:
-      step_operator.vertex:
-        accelerator_type: "NVIDIA_TESLA_P100"
-        accelerator_count: 1
-        machine_type: "n1-standard-8"
+#  generate_embeddings:
+#    step_operator: "terraform-gcp-6c0fd52233ca"
+#    settings:
+#      step_operator.vertex:
+#        accelerator_type: "NVIDIA_TESLA_P100"
+#        accelerator_count: 1
+#        machine_type: "n1-standard-8"
 
 # configuration of the Model Control Plane
 model:

llm-complete-guide/configs/rag_local_dev.yaml

@@ -13,6 +13,7 @@ settings:
       - psycopg2-binary
       - tiktoken
       - ratelimit
+      - rerankers
     environment:
       ZENML_PROJECT_SECRET_NAME: llm_complete
       ZENML_ENABLE_RICH_TRACEBACK: FALSE

llm-complete-guide/pipelines/llm_basic_rag.py

@@ -15,14 +15,15 @@
 # limitations under the License.
 #
 
+from zenml import pipeline
+
 from steps.populate_index import (
     generate_embeddings,
     index_generator,
     preprocess_documents,
 )
 from steps.url_scraper import url_scraper
 from steps.web_url_loader import web_url_loader
-from zenml import pipeline
 
 
 @pipeline

llm-complete-guide/pipelines/llm_eval.py

@@ -13,6 +13,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import os
+from pathlib import Path
+from typing import Optional
+
+import click
 
 from steps.eval_e2e import e2e_evaluation, e2e_evaluation_llm_judged
 from steps.eval_retrieval import (
@@ -64,3 +69,39 @@ def llm_eval() -> None:
         average_helpfulness_score,
         average_relevance_score,
     )
+
+
+@click.option(
+    "--no-cache",
+    "no_cache",
+    is_flag=True,
+    default=False,
+    help="Disable cache.",
+)
+@click.option(
+    "--config",
+    "config",
+    default="rag_local_dev.yaml",
+    help="Specify a configuration file"
+)
+def main(
+        no_cache: bool = False,
+        config: Optional[str] = "rag_eval.yaml"
+):
+    """
+    Executes the pipeline to train a basic RAG model.
+
+    Args:
+        no_cache (bool): If `True`, cache will be disabled.
+        config (str): The path to the configuration file.
+    """
+    config_path = Path(__file__).parent.parent / "configs" / config
+
+    llm_eval.with_options(
+        config_path=str(config_path),
+        enable_cache=not no_cache
+    )()
+
+
+if __name__ == "__main__":
+    main()

llm-complete-guide/run.py

@@ -208,7 +208,6 @@ def main(
             rag_deployment.with_options(
                 config_path=config_path, **pipeline_args
             )()
-
     if deploy:
         rag_deployment.with_options(**pipeline_args)()
     if evaluation:

llm-complete-guide/scripts/gh_action_rag.py

@@ -0,0 +1,106 @@
+# Apache Software License 2.0
+#
+# Copyright (c) ZenML GmbH 2024. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+from uuid import UUID
+
+import click
+from zenml.client import Client
+
+from pipelines import llm_basic_rag
+
+
+@click.command(
+    help="""
+ZenML LLM Complete - Rag Pipeline
+"""
+)
+@click.option(
+    "--no-cache",
+    "no_cache",
+    is_flag=True,
+    default=False,
+    help="Disable cache.",
+)
+
+@click.option(
+    "--create-template",
+    "create_template",
+    is_flag=True,
+    default=False,
+    help="Create a run template.",
+)
+@click.option(
+    "--config",
+    "config",
+    default="rag_local_dev.yaml",
+    help="Specify a configuration file"
+)
+@click.option(
+    "--action-id",
+    "action_id",
+    default=None,
+    help="Specify an action ID"
+)
+def main(
+    no_cache: bool = False,
+    config: Optional[str]= "rag_local_dev.yaml",
+    create_template: bool = False,
+    action_id: Optional[str] = None
+):
+    """
+    Executes the pipeline to train a basic RAG model.
+
+    Args:
+        no_cache (bool): If `True`, cache will be disabled.
+        config (str): The path to the configuration file.
+        create_template (bool): If `True`, a run template will be created.
+        action_id (str): The action ID.
+    """
+    client = Client()
+    config_path = Path(__file__).parent.parent / "configs" / config
+
+    if create_template:
+        # run pipeline
+        run = llm_basic_rag.with_options(
+            config_path=str(config_path),
+            enable_cache=not no_cache
+        )()
+        # create new run template
+        rt = client.create_run_template(
+            name=f"production-llm-complete-{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}",
+            deployment_id=run.deployment_id
+        )
+        # update the action with the new template
+        client.update_action(
+            name_id_or_prefix=UUID(action_id),
+            configuration={
+                "template_id": str(rt.id)
+            }
+        )
+
+    else:
+        llm_basic_rag.with_options(
+            config_path=str(config_path),
+            enable_cache=not no_cache
+        )()
+
+
+if __name__ == "__main__":
+    main()