Working as grid search

Author: AlexejPenner

native-experiment-tracking/analyze.py

@@ -0,0 +1,73 @@
+import numpy as np
+from matplotlib import pyplot as plt
+from zenml.client import Client
+
+
+def main():
+    client = Client()
+
+    model_versions = client.list_model_versions(model_name_or_id="breast_cancer_classifier", size=30, hydrate=True)
+
+    alpha_values = []
+    losses = []
+    penalties = []
+    test_accuracies = []
+    train_accuracies = []
+
+    for model_version in model_versions:
+        mv_metadata = model_version.run_metadata
+
+        alpha_values.append(mv_metadata.get("alpha_value", None).value)
+        losses.append(mv_metadata.get("loss", None).value)
+        penalties.append(mv_metadata.get("penalty", None).value)
+        test_accuracies.append(mv_metadata.get("test_accuracy", None).value)
+        train_accuracies.append(mv_metadata.get("train_accuracy", None).value)
+
+    generate_plot(alpha_values, losses, penalties, test_accuracies)
+
+
+def generate_plot(alpha_values, losses, penalties, test_accuracies):
+    # Convert losses and penalties to numerical indices
+    unique_losses = list(set(losses))
+    unique_penalties = list(set(penalties))
+
+    loss_indices = [unique_losses.index(loss) for loss in losses]
+    penalty_indices = [unique_penalties.index(penalty) for penalty in penalties]
+
+    # Create a figure and a 3D axis
+    fig = plt.figure(figsize=(12, 8))
+    ax = fig.add_subplot(111, projection='3d')
+
+    # Create a scatter plot
+    scatter = ax.scatter(alpha_values, loss_indices, penalty_indices, c=test_accuracies, cmap='viridis')
+
+    # Set labels for each axis
+    ax.set_xlabel('Alpha')
+    ax.set_ylabel('Loss')
+    ax.set_zlabel('Penalty')
+
+    # Set custom ticks for loss and penalty axes
+    ax.set_yticks(range(len(unique_losses)))
+    ax.set_yticklabels(unique_losses)
+    ax.set_zticks(range(len(unique_penalties)))
+    ax.set_zticklabels(unique_penalties)
+
+    # Add a color bar
+    cbar = plt.colorbar(scatter)
+    cbar.set_label('Accuracy')
+
+    # Set a title
+    plt.title('Accuracy vs. Alpha, Loss, and Penalty')
+
+    # Adjust the viewing angle
+    ax.view_init(elev=20, azim=45)
+
+    # Show the plot
+    plt.tight_layout()
+    plt.show()
+    return
+
+
+
+if __name__ == "__main__":
+    main()

native-experiment-tracking/configs/training.yaml

@@ -9,10 +9,3 @@ settings:
       - matplotlib
       - pillow
       - numpy
-
-# configuration of the Model Control Plane
-model:
-  name: breast_cancer_classifier
-  license: Apache 2.0
-  description: A breast cancer classifier
-  tags: ["breast_cancer", "classifier"]

native-experiment-tracking/pipelines/training.py

@@ -18,7 +18,7 @@
 from typing import Optional
 from uuid import UUID
 
-from steps import model_evaluator, model_promoter, model_trainer, model_grid_searcher
+from steps import model_evaluator, model_promoter, model_trainer
 
 from pipelines import (
     feature_engineering,
@@ -33,8 +33,9 @@
 
 @pipeline
 def training(
-    train_dataset_id: Optional[UUID] = None,
-    test_dataset_id: Optional[UUID] = None,
+    alpha_value: float,
+    penalty: str,
+    loss: str,
     target: Optional[str] = "target",
 ):
     """
@@ -47,27 +48,19 @@ def training(
     model version.
 
     Args:
-        train_dataset_id: ID of the train dataset produced by feature engineering.
-        test_dataset_id: ID of the test dataset produced by feature engineering.
         target: Name of target column in dataset.
+        alpha_value: Alpha value to use for the train step,
+        penalty: Penalty to use for sgd,
+        loss: Loss function to be used for sgd,
     """
     # Link all the steps together by calling them and passing the output
     # of one step as the input of the next step.
 
     # Execute Feature Engineering Pipeline
-    if train_dataset_id is None or test_dataset_id is None:
-        dataset_trn, dataset_tst = feature_engineering()
-    else:
-        client = Client()
-        dataset_trn = client.get_artifact_version(
-            name_id_or_prefix=train_dataset_id
-        )
-        dataset_tst = client.get_artifact_version(
-            name_id_or_prefix=test_dataset_id
-        )
+    dataset_trn, dataset_tst = feature_engineering()
 
-    model, _, _ = model_grid_searcher(
-        dataset_trn=dataset_trn, target=target
+    model, _ = model_trainer(
+        dataset_trn=dataset_trn, target=target, alpha_value=alpha_value, penalty=penalty, loss=loss
     )
 
     acc, _ = model_evaluator(
@@ -76,5 +69,3 @@ def training(
         dataset_tst=dataset_tst,
         target=target,
     )
-
-    model_promoter(accuracy=acc)

native-experiment-tracking/run.py

@@ -17,21 +17,24 @@
 import os
 
 import click
-
+from sklearn.utils._param_validation import InvalidParameterError
+from zenml import Model
+from zenml.client import Client
 from zenml.logger import get_logger
 
 from pipelines import training
 
 logger = get_logger(__name__)
 
+
 @click.option(
     "--no-cache",
     is_flag=True,
     default=False,
     help="Disable caching for the pipeline run.",
 )
 def main(
-    no_cache: bool = False,
+        no_cache: bool = False,
 ):
     """Main entry point for the pipeline execution.
 
@@ -45,20 +48,34 @@ def main(
     Args:
         no_cache: If `True` cache will be disabled.
     """
-    config_folder = os.path.join(
+    client = Client()
+    config_path = os.path.join(
         os.path.dirname(os.path.realpath(__file__)),
         "configs",
+        "training.yaml"
     )
+    enable_cache = not no_cache
+
+    alpha_values = [0.0001, 0.001, 0.01]
+    penalties = ["l2", "l1", "elasticnet"]
+    losses = ["hinge", "squared_hinge", "modified_huber"]
+    for penalty in penalties:
+        for loss in losses:
+            for alpha_value in alpha_values:
+                logger.info(f"Training with alpha: {alpha_value}, penalty: {penalty}, loss: {loss}")
+
+                model = Model(
+                    name="breast_cancer_classifier",
+                    tags=[f"alpha: {alpha_value}", f"penalty: {penalty}", f"loss: {loss}"]
+                )
+                try:
+                    training.with_options(config_path=config_path, enable_cache=enable_cache, model=model)(
+                        alpha_value=alpha_value, penalty=penalty, loss=loss)
+                except RuntimeError:
+                    pass
+                else:
+                    logger.info("Training pipeline finished successfully!\n\n")
 
-    pipeline_args = {}
-    if no_cache:
-        pipeline_args["enable_cache"] = False
-    pipeline_args["config_path"] = os.path.join(
-        config_folder, "training.yaml"
-    )
-    training.with_options(**pipeline_args)()
-    training.with_options(**pipeline_args)()
-    logger.info("Training pipeline finished successfully!\n\n")
 
 if __name__ == "__main__":
     main()

native-experiment-tracking/steps/__init__.py

@@ -33,6 +33,3 @@
 from .model_trainer import (
     model_trainer,
 )
-from .model_grid_search import (
-    model_grid_searcher
-)