More fine grained control over configs

Author: AlexejPenner

.github/workflows/run_complete_llm.yml

@@ -52,4 +52,4 @@ jobs:
       - name: Run pipeline (Staging)
         working-directory: ./llm-complete-guide
         run: |
-          python run.py --rag --evaluation --no-cache
+          python pipelines/llm_basic_rag.py --config=rag_gcp_yaml --no-cache

llm-complete-guide/configs/rag_gcp.yaml

@@ -11,6 +11,7 @@ settings:
       - psycopg2-binary
       - tiktoken
       - ratelimit
+      - rerankers
     environment:
       ZENML_PROJECT_SECRET_NAME: llm_complete
       ZENML_ENABLE_RICH_TRACEBACK: FALSE
@@ -20,13 +21,13 @@ steps:
   url_scraper:
     parameters:
       docs_url: https://docs.zenml.io
-  generate_embeddings:
-    step_operator: "terraform-gcp-6c0fd52233ca"
-    settings:
-      step_operator.vertex:
-        accelerator_type: "NVIDIA_TESLA_P100"
-        accelerator_count: 1
-        machine_type: "n1-standard-8"
+#  generate_embeddings:
+#    step_operator: "terraform-gcp-6c0fd52233ca"
+#    settings:
+#      step_operator.vertex:
+#        accelerator_type: "NVIDIA_TESLA_P100"
+#        accelerator_count: 1
+#        machine_type: "n1-standard-8"
 
 # configuration of the Model Control Plane
 model:

llm-complete-guide/configs/rag_local_dev.yaml

@@ -13,6 +13,7 @@ settings:
       - psycopg2-binary
       - tiktoken
       - ratelimit
+      - rerankers
     environment:
       ZENML_PROJECT_SECRET_NAME: llm_complete
       ZENML_ENABLE_RICH_TRACEBACK: FALSE

llm-complete-guide/pipelines/llm_basic_rag.py

@@ -14,6 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import os
+from typing import Optional
+
+import click
+from litellm import enable_cache
 
 from steps.populate_index import (
     generate_embeddings,
@@ -41,3 +46,35 @@ def llm_basic_rag() -> None:
     processed_docs = preprocess_documents(documents=docs)
     embedded_docs = generate_embeddings(split_documents=processed_docs)
     index_generator(documents=embedded_docs)
+
+
+@click.option(
+    "--no-cache",
+    "no_cache",
+    is_flag=True,
+    default=False,
+    help="Disable cache.",
+)
+@click.option(
+    "--config",
+    "config",
+    default="rag_local_dev.yaml",
+    help="Specify a configuration file"
+)
+def main(
+    no_cache: bool,
+    config: Optional[str]= "rag_local_dev.yaml"
+):
+    config_path = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)),
+        "configs",
+        config,
+    )
+    llm_basic_rag.with_options(
+        config_path=config_path,
+        enable_cache=not no_cache
+    )()
+
+
+if __name__ == "__main__":
+    main()