Merge branch 'main' into feature/vertex

Author: safoinme

.github/workflows/production_run_complete_llm.yml

@@ -11,20 +11,21 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run-staging-workflow:
+  run-production-workflow:
     runs-on: ubuntu-latest
     env:
-      ZENML_HOST: ${{ secrets.ZENML_HOST }}
-      ZENML_API_KEY: ${{ secrets.ZENML_API_KEY }}
-      ZENML_PRODUCTION_STACK : 51a49786-b82a-4646-bde7-a460efb0a9c5
+      ZENML_STORE_URL: ${{ secrets.ZENML_PROJECTS_HOST }}
+      ZENML_STORE_API_KEY: ${{ secrets.ZENML_PROJECTS_API_KEY }}
+      ZENML_PRODUCTION_STACK: b3951d43-0fb2-4d32-89c5-3399374e7c7e # Set this to your production stack ID
       ZENML_GITHUB_SHA: ${{ github.event.pull_request.head.sha }}
       ZENML_GITHUB_URL_PR: ${{ github.event.pull_request._links.html.href }}
       ZENML_DEBUG: true
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_LOGGING_VERBOSITY: INFO
       ZENML_PROJECT_SECRET_NAME: llm-complete
       ZENML_DISABLE_CLIENT_SERVER_MISMATCH_WARNING: True
-      ZENML_ACTION_ID: 23a4d58c-bd2b-47d5-a41d-0a845d2982f8
+      ZENML_EVENT_SOURCE_ID: ae6ae536-d811-4838-a44b-744b768a0f31  # Set this to your preferred event source ID
+      ZENML_SERVICE_ACCOUNT_ID: fef76af2-382f-4ab2-9e6b-5eb85a303f0e  # Set this to your service account ID or delete
 
     steps:
       - name: Check out repository code
@@ -37,15 +38,15 @@ jobs:
       - name: Install requirements
         working-directory: ./llm-complete-guide
         run: |
-          pip3 install -r requirements.txt
-          pip3 install -r requirements-argilla.txt
-          zenml integration install gcp -y
+          pip3 install uv
+          uv pip install -r requirements.txt --system
+          uv pip install -r requirements-argilla.txt --system
+          zenml integration install gcp -y --uv
 
       - name: Connect to ZenML server
         working-directory: ./llm-complete-guide
         run: |
           zenml init
-          zenml connect --url $ZENML_HOST --api-key $ZENML_API_KEY
 
       - name: Set stack (Production)
         working-directory: ./llm-complete-guide
@@ -55,4 +56,4 @@ jobs:
       - name: Run pipeline, create pipeline, configure trigger (Production)
         working-directory: ./llm-complete-guide
         run: |
-          python gh_action_rag.py --no-cache --create-template --action-id  ${{ env.ZENML_ACTION_ID }} --config rag_gcp.yaml
+          python gh_action_rag.py --no-cache --create-template --event-source-id ${{ env.ZENML_EVENT_SOURCE_ID }} --service-account-id ${{ env.ZENML_SERVICE_ACCOUNT_ID }} --config production/rag.yaml --zenml-model-version production

.github/workflows/staging_run_complete_llm.yml

@@ -12,9 +12,9 @@ jobs:
   run-staging-workflow:
     runs-on: ubuntu-latest
     env:
-      ZENML_HOST: ${{ secrets.ZENML_HOST }}
-      ZENML_API_KEY: ${{ secrets.ZENML_API_KEY }}
-      ZENML_STAGING_STACK: 51a49786-b82a-4646-bde7-a460efb0a9c5
+      ZENML_STORE_URL: ${{ secrets.ZENML_PROJECTS_HOST }}
+      ZENML_STORE_API_KEY: ${{ secrets.ZENML_PROJECTS_API_KEY }}
+      ZENML_STAGING_STACK : 67166d73-a44e-42f9-b67f-011e9afab9b5 # Set this to your staging stack ID
       ZENML_GITHUB_SHA: ${{ github.event.pull_request.head.sha }}
       ZENML_GITHUB_URL_PR: ${{ github.event.pull_request._links.html.href }}
       ZENML_DEBUG: true
@@ -34,15 +34,15 @@ jobs:
       - name: Install requirements
         working-directory: ./llm-complete-guide
         run: |
-          pip3 install -r requirements.txt
-          pip3 install -r requirements-argilla.txt
-          zenml integration install gcp -y
+          pip3 install uv
+          uv pip install -r requirements.txt --system
+          uv pip install -r requirements-argilla.txt --system
+          zenml integration install aws s3 -y --uv
 
       - name: Connect to ZenML server
         working-directory: ./llm-complete-guide
         run: |
           zenml init
-          zenml connect --url $ZENML_HOST --api-key $ZENML_API_KEY
 
       - name: Set stack (Staging)
         working-directory: ./llm-complete-guide
@@ -52,4 +52,4 @@ jobs:
       - name: Run pipeline (Staging)
         working-directory: ./llm-complete-guide
         run: |
-          python gh_action_rag.py --no-cache --config rag_gcp.yaml
+          python gh_action_rag.py --no-cache --config staging/rag.yaml --zenml-model-version staging

classifier-e2e/README.md

@@ -11,58 +11,76 @@ pinned: false
 license: apache-2.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# ZenML MLOps Breast Cancer Classification Demo
 
-# 📜 ZenML Stack Show Case
+## 🌍 Project Overview
 
-This project aims to demonstrate the power of stacks. The code in this 
-project assumes that you have quite a few stacks registered already. 
+This is a minimalistic MLOps project demonstrating how to put machine learning 
+workflows into production using ZenML. The project focuses on building a breast 
+cancer classification model with end-to-end ML pipeline management.
 
-## default
-  * `default` Orchestrator
-  * `default` Artifact Store
+### Key Features
 
-```commandline
-zenml stack set default
-python run.py --training-pipeline
+- 🔬 Feature engineering pipeline
+- 🤖 Model training pipeline
+- 🧪 Batch inference pipeline
+- 📊 Artifact and model lineage tracking
+- 🔗 Integration with Weights & Biases for experiment tracking
+
+## 🚀 Installation
+
+1. Clone the repository
+2. Install requirements:
+    ```bash
+    pip install -r requirements.txt
+    ```
+3. Install ZenML integrations:
+    ```bash
+    zenml integration install sklearn xgboost wandb -y
+    zenml login
+    zenml init
+    ```
+4. You need to register a stack with a [Weights & Biases Experiment Tracker](https://docs.zenml.io/stack-components/experiment-trackers/wandb). 
+
+## 🧠 Project Structure
+
+- `steps/`: Contains individual pipeline steps
+- `pipelines/`: Pipeline definitions
+- `run.py`: Main script to execute pipelines
+
+## 🔍 Workflow and Execution
+
+First, you need to set your stack:
+
+```bash
+zenml stack set stack-with-wandb
 ```
 
-## local-sagemaker-step-operator-stack
-  * `default` Orchestrator
-  * `s3` Artifact Store
-  * `local` Image Builder
-  * `aws` Container Registry
-  * `Sagemaker` Step Operator
+### 1. Data Loading and Feature Engineering
 
-```commandline
-zenml stack set local-sagemaker-step-operator-stack
-zenml integration install aws -y
-python run.py --training-pipeline
+- Uses the Breast Cancer dataset from scikit-learn
+- Splits data into training and inference sets
+- Preprocesses data for model training
+
+```bash
+python run.py --feature-pipeline
 ```
 
-## sagemaker-airflow-stack
-  * `Airflow` Orchestrator
-  * `s3` Artifact Store
-  * `local` Image Builder
-  * `aws` Container Registry
-  * `Sagemaker` Step Operator
-
-```commandline
-zenml stack set sagemaker-airflow-stack
-zenml integration install airflow -y
-pip install apache-airflow-providers-docker apache-airflow~=2.5.0
-zenml stack up
+### 2. Model Training
+
+- Supports multiple model types (SGD, XGBoost)
+- Evaluates and compares model performance
+- Tracks model metrics with Weights & Biases
+
+```bash
 python run.py --training-pipeline
 ```
 
-## sagemaker-stack
-  * `Sagemaker` Orchestrator
-  * `s3` Artifact Store
-  * `local` Image Builder
-  * `aws` Container Registry
-  * `Sagemaker` Step Operator
+### 3. Batch Inference
 
-```commandline
-zenml stack set sagemaker-stack
-python run.py --training-pipeline
+- Loads production model
+- Generates predictions on new data
+
+```bash
+python run.py --inference-pipeline
 ```

classifier-e2e/requirements.txt

@@ -1,4 +1,4 @@
-zenml[server]>=0.55.2
+zenml[server]>=0.70.0
 notebook
 scikit-learn<1.3
 s3fs>2022.3.0,<=2023.4.0

classifier-e2e/run_full.ipynb

@@ -38,7 +38,7 @@
    "source": [
     "! pip3 install -r requirements.txt\n",
     "! zenml integration install sklearn xgboost -y\n",
-    "! zenml connect --url https://1cf18d95-zenml.cloudinfra.zenml.io \n",
+    "! zenml login https://1cf18d95-zenml.cloudinfra.zenml.io \n",
     "\n",
     "import IPython\n",
     "IPython.Application.instance().kernel.do_shutdown(restart=True)"
@@ -941,10 +941,17 @@
     "        .ravel()\n",
     "        .tolist(),\n",
     "    }\n",
-    "    log_model_metadata(metadata={\"wandb_url\": wandb.run.url})\n",
-    "    log_artifact_metadata(\n",
+    "\n",
+    "    try:\n",
+    "        if get_step_context().model:\n",
+    "            log_metadata(metadata=metadata, infer_model=True)\n",
+    "    except StepContextError:\n",
+    "        # If a model is not configured, it is not able to log metadata\n",
+    "        pass\n",
+    "\n",
+    "    log_metadata(\n",
     "        metadata=metadata,\n",
-    "        artifact_name=\"breast_cancer_classifier\",\n",
+    "        artifact_version_id=get_step_context().inputs[\"model\"].id,\n",
     "    )\n",
     "\n",
     "    wandb.log({\"train_accuracy\": metadata[\"train_accuracy\"]})\n",
@@ -1073,6 +1080,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "1e2130b9",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1083,6 +1091,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "476cbf5c",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1091,6 +1100,7 @@
   },
   {
    "cell_type": "markdown",
+   "id": "75df10e7",
    "metadata": {},
    "source": [
     "Now full run executed on local stack and experiment is tracked using Model Control Plane and Weights&Biases.\n",
@@ -1103,6 +1113,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "bfd6345f",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1113,6 +1124,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "24358031",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1136,7 +1148,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.18"
+   "version": "3.11.3"
   }
  },
  "nbformat": 4,

classifier-e2e/run_skip_basics.ipynb

@@ -38,7 +38,7 @@
    "source": [
     "! pip3 install -r requirements.txt\n",
     "! zenml integration install sklearn xgboost -y\n",
-    "! zenml connect --url https://1cf18d95-zenml.cloudinfra.zenml.io \n",
+    "! zenml login https://1cf18d95-zenml.cloudinfra.zenml.io \n",
     "\n",
     "import IPython\n",
     "IPython.Application.instance().kernel.do_shutdown(restart=True)"
@@ -829,10 +829,17 @@
     "        .ravel()\n",
     "        .tolist(),\n",
     "    }\n",
-    "    log_model_metadata(metadata={\"wandb_url\": wandb.run.url})\n",
-    "    log_artifact_metadata(\n",
+    "\n",
+    "    try:\n",
+    "        if get_step_context().model:\n",
+    "            log_metadata(metadata=metadata, infer_model=True)\n",
+    "    except StepContextError:\n",
+    "        # If a model is not configured, it is not able to log metadata\n",
+    "        pass\n",
+    "\n",
+    "    log_metadata(\n",
     "        metadata=metadata,\n",
-    "        artifact_name=\"breast_cancer_classifier\",\n",
+    "        artifact_version_id=get_step_context().inputs[\"model\"].id,\n",
     "    )\n",
     "\n",
     "    wandb.log({\"train_accuracy\": metadata[\"train_accuracy\"]})\n",
@@ -1211,7 +1218,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.18"
+   "version": "3.11.3"
   }
  },
  "nbformat": 4,

classifier-e2e/steps/deploy_endpoint.py

@@ -7,6 +7,7 @@
 from utils.aws import get_aws_config
 from utils.sagemaker_materializer import SagemakerPredictorMaterializer
 from zenml import ArtifactConfig, get_step_context, log_artifact_metadata, step
+from zenml.enums import ArtifactType
 
 
 @step(
@@ -16,7 +17,10 @@
 def deploy_endpoint() -> (
     Annotated[
         Predictor,
-        ArtifactConfig(name="sagemaker_endpoint", is_deployment_artifact=True),
+        ArtifactConfig(
+            name="sagemaker_endpoint",
+            artifact_type=ArtifactType.SERVICE
+        ),
     ]
 ):
     role, session, region = get_aws_config()