Externalize image registry URL configuration

3-prod_datascience/evaluate_model.py

@@ -17,9 +17,9 @@ def evaluate_keras_model_performance(
     scaler: Input[Artifact],
     label_encoder: Input[Artifact],
     model_name: str,
-    cluster_domain: str,
     version: str,
-    prod_flag: bool,
+    model_registry_server_address: str,
+    model_registry_port: int,
     metrics: Output[Metrics],
     classification_metrics: Output[ClassificationMetrics]
 ):
@@ -47,15 +47,10 @@ def evaluate_keras_model_performance(
     y_test_argmax = np.argmax(y_test, axis=1)
 
     accuracy = np.sum(y_pred_argmax == y_test_argmax) / len(y_pred_argmax)
-
-    # Get the previous models properties from the Model Registry
-    if prod_flag:
-        namespace = environ.get("NAMESPACE").split("-")[0]+"-prod"
-    else:
-        namespace = environ.get("NAMESPACE").split("-")[0]
 
-    environ["KF_PIPELINES_SA_TOKEN_PATH"] = "/var/run/secrets/kubernetes.io/serviceaccount/token" # Hotfix to access the endpoint
-    registry = ModelRegistry(server_address=f"https://{namespace}-registry-rest.{cluster_domain}", port=443, author="someone", is_secure=False)
+    environ["KF_PIPELINES_SA_TOKEN_PATH"] = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+    registry = ModelRegistry(server_address=model_registry_server_address, port=model_registry_port, author="someone",is_secure=False)
+
     previous_model_properties = {}
 
     #Wrap with try except to see if the model exists in the registry
@@ -119,4 +114,4 @@ def validate_onnx_model(
     for rt_res, keras_res in zip(onnx_pred[0], keras_pred):
         np.testing.assert_allclose(rt_res, keras_res, rtol=1e-5, atol=1e-5)
 
-    print("Results match")
+    print("Results match")

3-prod_datascience/prod_train_save_pipeline.py

@@ -1,4 +1,3 @@
-# kfp imports
 import kfp
 import kfp.dsl as dsl
 from kfp.dsl import (
@@ -30,7 +29,7 @@
   name='kfp-training-pipeline',
   description='We train an amazing model 🚂'
 )
-def training_pipeline(hyperparameters: dict, model_name: str, version: str, cluster_domain: str, model_storage_pvc: str, prod_flag: bool):
+def training_pipeline(hyperparameters: dict, model_name: str, version: str, model_storage_pvc: str, prod_flag: bool, model_registry_server_address: str, model_registry_port: int):
     ### 🐶 Fetch Data from GitHub
     fetch_task = fetch_data()
 
@@ -63,9 +62,9 @@ def training_pipeline(hyperparameters: dict, model_name: str, version: str, clus
         scaler = pre_processing_task.outputs["scaler"],
         label_encoder = pre_processing_task.outputs["label_encoder"],
         model_name = model_name,
-        cluster_domain = cluster_domain,
-        version = version, # Add version to force a rerun of this step every new version
-        prod_flag = prod_flag,
+        version = version,
+        model_registry_server_address = model_registry_server_address,
+        model_registry_port = model_registry_port,
     )
     kubernetes.use_field_path_as_env(
         model_evaluation_task,
@@ -85,8 +84,9 @@ def training_pipeline(hyperparameters: dict, model_name: str, version: str, clus
     register_model_task = push_to_model_registry(
         model_name = model_name, 
         version = version,
-        cluster_domain = cluster_domain,
         prod_flag = prod_flag,
+        model_registry_server_address = model_registry_server_address,
+        model_registry_port = model_registry_port,
         keras_model = training_task.outputs["trained_model"],
         model = convert_task.outputs["onnx_model"],
         metrics = model_evaluation_task.outputs["metrics"],
@@ -122,7 +122,6 @@ def training_pipeline(hyperparameters: dict, model_name: str, version: str, clus
         },
         "model_name": "jukebox",
         "version": "0.0.2",
-        "cluster_domain": "<CLUSTER_DOMAIN>", # 👈 add your cluster domain here
         "model_storage_pvc": "jukebox-model-pvc",
         "prod_flag": False
     }
@@ -154,4 +153,4 @@ def training_pipeline(hyperparameters: dict, model_name: str, version: str, clus
         arguments=metadata,
         experiment_name="kfp-training-pipeline",
         enable_caching=True
-    )
+    )

3-prod_datascience/save_model.py

@@ -14,8 +14,9 @@
 def push_to_model_registry(
     model_name: str,
     version: str,
-    cluster_domain: str,
     prod_flag: bool,
+    model_registry_server_address: str,
+    model_registry_port: int,
     keras_model: Input[Artifact],
     model: Input[Artifact],
     metrics: Input[Metrics],
@@ -82,24 +83,14 @@ def _do_upload(s3_client, model_path, object_name, s3_bucket_name):
     environ["KF_PIPELINES_SA_TOKEN_PATH"] = "/var/run/secrets/kubernetes.io/serviceaccount/token"
 
     ############ Register to Model Registry ############
-    namespace_file_path =\
-        '/var/run/secrets/kubernetes.io/serviceaccount/namespace'
-    with open(namespace_file_path, 'r') as namespace_file:
-        namespace = namespace_file.read()
-
-    if prod_flag:
-        namespace = namespace.split("-")[0]+"-prod"
-    else:
-        namespace = namespace.split("-")[0]
 
     model_object_prefix = model_name if model_name else "model"
     version = version if version else datetime.now().strftime('%y%m%d%H%M')
-    server_address = f"https://{namespace}-registry-rest.{cluster_domain}"
 
     registry = ModelRegistry(
-        server_address=server_address,
-        port=443,
-        author=namespace,
+        server_address=model_registry_server_address,
+        port=model_registry_port,
+        author="someone",
         is_secure=False
     )
     registered_model_name = model_object_prefix