Misc features/bugfixes for RANO

cli/medperf/commands/compatibility_test/utils.py

@@ -138,8 +138,13 @@ def create_test_dataset(
     )
     data_creation.validate()
     data_creation.create_dataset_object()
-    # TODO: existing dataset could make problems
+
     # make some changes since this is a test dataset
+    remove_path(data_creation.dataset.data_path)
+    remove_path(data_creation.dataset.labels_path)
+    remove_path(data_creation.dataset.metadata_path)
+    remove_path(data_creation.dataset.report_path)
+    remove_path(data_creation.dataset.statistics_path)
     config.tmp_paths.remove(data_creation.dataset.path)
     if skip_data_preparation_step:
         data_creation.make_dataset_prepared()

cli/medperf/commands/dataset/prepare.py

@@ -208,6 +208,8 @@ def run_sanity_check(self):
             "data_path": out_datapath,
             "labels_path": out_labelspath,
         }
+        if self.metadata_specified:
+            sanity_check_mounts["metadata_path"] = self.metadata_path
 
         self.ui.text = "Running sanity check..."
         try:

cli/medperf/config.py

@@ -171,6 +171,7 @@
 cube_metadata_filename = "mlcube-meta.yaml"
 log_file = "medperf.log"
 webui_log_file = "medperf_webui.log"
+data_monitor_log_file = "medperf_data_monitor.log"
 log_package_file = "medperf_logs.tar.gz"
 tarball_filename = "tmp.tar.gz"
 demo_dset_paths_file = "paths.yaml"

cli/medperf/containers/runners/docker_runner.py

@@ -1,4 +1,3 @@
-from medperf.exceptions import InvalidContainerSpec
 from .utils import (
     run_command,
     check_allowed_run_args,
@@ -7,6 +6,7 @@
     add_medperf_environment_variables,
     add_network_config,
     add_medperf_tmp_folder,
+    check_docker_image_hash,
 )
 from .runner import Runner
 import logging
@@ -22,13 +22,15 @@ def download(
         expected_image_hash,
         download_timeout: int = None,
         get_hash_timeout: int = None,
+        alternative_image_hash: str = None,
     ):
         docker_image = self.parser.get_setup_args()
         command = ["docker", "pull", docker_image]
         run_command(command, timeout=download_timeout)
         computed_image_hash = get_docker_image_hash(docker_image, get_hash_timeout)
-        if expected_image_hash and expected_image_hash != computed_image_hash:
-            raise InvalidContainerSpec("hash mismatch")
+        check_docker_image_hash(
+            computed_image_hash, expected_image_hash, alternative_image_hash
+        )
         return computed_image_hash
 
     def run(

cli/medperf/containers/runners/runner.py

@@ -8,6 +8,7 @@ def download(
         expected_image_hash: str,
         download_timeout: int = None,
         get_hash_timeout: int = None,
+        alternative_image_hash: str = None,
     ):
         pass
 

cli/medperf/containers/runners/singularity_runner.py

@@ -1,9 +1,5 @@
 from medperf.comms.entity_resources import resources
-from medperf.exceptions import (
-    InvalidContainerSpec,
-    InvalidArgumentError,
-    MedperfException,
-)
+from medperf.exceptions import InvalidArgumentError, MedperfException
 from medperf.utils import remove_path
 from .utils import (
     run_command,
@@ -13,6 +9,7 @@
     add_medperf_environment_variables,
     add_network_config,
     add_medperf_tmp_folder,
+    check_docker_image_hash,
 )
 from .singularity_utils import (
     get_docker_image_hash_from_dockerhub,
@@ -54,6 +51,7 @@ def download(
         expected_image_hash,
         download_timeout: int = None,
         get_hash_timeout: int = None,
+        alternative_image_hash: str = None,
     ):
         if self.parser.container_type == "SingularityFile":
             return self._download_singularity_file(
@@ -66,6 +64,7 @@ def download(
                 expected_image_hash,
                 download_timeout,
                 get_hash_timeout,
+                alternative_image_hash,
             )
 
     def _download_singularity_file(
@@ -86,13 +85,15 @@ def _download_and_convert_docker_image(
         expected_image_hash,
         download_timeout: int = None,
         get_hash_timeout: int = None,
+        alternative_image_hash: str = None,
     ):
         docker_image = self.parser.get_setup_args()
         computed_image_hash = get_docker_image_hash_from_dockerhub(
             docker_image, get_hash_timeout
         )
-        if expected_image_hash and expected_image_hash != computed_image_hash:
-            raise InvalidContainerSpec("hash mismatch")
+        check_docker_image_hash(
+            computed_image_hash, expected_image_hash, alternative_image_hash
+        )
 
         sif_image_folder = os.path.join(
             self.container_files_base_path, config.image_path

cli/medperf/containers/runners/utils.py

@@ -102,3 +102,22 @@ def add_medperf_tmp_folder(output_volumes, tmp_folder):
     output_volumes.append(
         {"host_path": tmp_folder, "mount_path": "/tmp", "type": "directory"}
     )
+
+
+def check_docker_image_hash(
+    computed_image_hash, expected_image_hash=None, alternative_image_hash=None
+):
+    if expected_image_hash and expected_image_hash != computed_image_hash:
+        # try with digest if possible
+        # This fixes an issue with newer docker versions where inspect returns
+        # the digest instead of the image ID. The cleaner fix will require changing how
+        # we define the image hash.
+        if alternative_image_hash is None:
+            raise InvalidContainerSpec(
+                f"Hash mismatch. Expected {expected_image_hash}, found {computed_image_hash}."
+            )
+        if alternative_image_hash != computed_image_hash:
+            raise InvalidContainerSpec(
+                f"Hash mismatch. Expected {expected_image_hash} or"
+                f" {alternative_image_hash}, found {computed_image_hash}."
+            )

cli/medperf/entities/cube.py

@@ -173,11 +173,15 @@ def download_run_files(self):
         except InvalidEntityError as e:
             raise InvalidEntityError(f"Container {self.name} additional files: {e}")
 
+        alternative_image_hash = None
+        if self.metadata is not None:
+            alternative_image_hash = self.metadata.get("digest", None)
         try:
             self.image_hash = self.runner.download(
                 expected_image_hash=self.image_hash,
                 download_timeout=config.mlcube_configure_timeout,
                 get_hash_timeout=config.mlcube_inspect_timeout,
+                alternative_image_hash=alternative_image_hash,
             )
         except InvalidEntityError as e:
             raise InvalidEntityError(f"Container {self.name} image: {e}")

cli/medperf/init.py

@@ -12,14 +12,15 @@
 from medperf.ui.factory import UIFactory
 
 
-def initialize(for_webui=False):
+def initialize(for_webui=False, for_data_monitor=False):
     if not for_webui:
         log_file_name = config.log_file
         ui_class = config.ui
     else:
         log_file_name = config.webui_log_file
         ui_class = config.webui
-
+    if for_data_monitor:
+        log_file_name = config.data_monitor_log_file
     # Apply any required migration
     apply_configuration_migrations()
 

cli/medperf/web_ui/app.py

@@ -21,7 +21,7 @@
 from medperf.web_ui.events import router as events_router
 from medperf.web_ui.medperf_login import router as medperf_login
 from medperf.web_ui.profiles import router as profiles_router
-from medperf.web_ui.auth import security_token, wrap_openapi, NotAuthenticatedException
+from medperf.web_ui.auth import wrap_openapi, NotAuthenticatedException
 
 web_app = FastAPI()
 
@@ -79,14 +79,6 @@ def startup_event():
     logging.getLogger("requests").setLevel(loglevel)
     log_machine_details()
 
-    # print security token to CLI (avoid logging to file)
-    print("=" * 40)
-    print()
-    print("Use security token to view the web-UI:")
-    print(security_token)
-    print()
-    print("=" * 40)
-
 
 @web_app.exception_handler(NotAuthenticatedException)
 def not_authenticated_exception_handler(

cli/medperf/web_ui/benchmarks/routes.py

@@ -14,7 +14,6 @@
 from medperf.entities.cube import Cube
 from medperf.account_management import get_medperf_user_data
 from medperf.entities.execution import Execution
-from medperf.exceptions import MedperfException
 from medperf.web_ui.common import (
     add_notification,
     check_user_api,
@@ -174,7 +173,7 @@ def test_benchmark(
         return_response["status"] = "success"
         return_response["results"] = results
         notification_message = "Benchmark workflow test succeeded"
-    except MedperfException as exp:
+    except Exception as exp:
         return_response["status"] = "failed"
         return_response["error"] = str(exp)
         notification_message = "Benchmark workflow test failed"
@@ -225,7 +224,7 @@ def register_benchmark(
         return_response["status"] = "success"
         return_response["benchmark_id"] = benchmark_id
         notification_message = "Benchmark successfully registered!"
-    except MedperfException as exp:
+    except Exception as exp:
         return_response["status"] = "failed"
         return_response["error"] = str(exp)
         notification_message = "Failed to register benchmark"
@@ -261,7 +260,7 @@ def approve(
         )
         return_response["status"] = "success"
         notification_message = "Association successfully approved"
-    except MedperfException as exp:
+    except Exception as exp:
         return_response["status"] = "failed"
         return_response["status"] = str(exp)
         notification_message = "Failed to approve association"
@@ -297,7 +296,7 @@ def reject(
         )
         return_response["status"] = "success"
         notification_message = "Association successfully rejected"
-    except MedperfException as exp:
+    except Exception as exp:
         return_response["status"] = "failed"
         return_response["status"] = str(exp)
         notification_message = "Failed to reject association"
@@ -336,7 +335,7 @@ def update_associations_policy(
         )
         return_response["status"] = "success"
         notification_message = "Associations policy updated"
-    except MedperfException as exp:
+    except Exception as exp:
         return_response["status"] = "failed"
         return_response["error"] = str(exp)
         notification_message = "Failed to update associations policy"

cli/medperf/web_ui/containers/routes.py

@@ -10,7 +10,6 @@
 from medperf.account_management import get_medperf_user_data
 from medperf.entities.cube import Cube
 from medperf.entities.benchmark import Benchmark
-from medperf.exceptions import MedperfException
 from medperf.web_ui.common import (
     add_notification,
     check_user_api,
@@ -139,7 +138,7 @@ def register_container(
         return_response["status"] = "success"
         return_response["container_id"] = container_id
         notification_message = "Container successfully registered"
-    except MedperfException as exp:
+    except Exception as exp:
         return_response["status"] = "failed"
         return_response["error"] = str(exp)
         notification_message = "Failed to register container"
@@ -172,7 +171,7 @@ def test_container(
         return_response["status"] = "success"
         return_response["results"] = results
         notification_message = "Container compatibility test succeeded!"
-    except MedperfException as exp:
+    except Exception as exp:
         return_response["status"] = "failed"
         return_response["error"] = str(exp)
         notification_message = "Container compatibility test failed"
@@ -202,7 +201,7 @@ def associate(
         AssociateCube.run(cube_uid=container_id, benchmark_uid=benchmark_id)
         return_response["status"] = "success"
         notification_message = "Successfully requested container association!"
-    except MedperfException as exp:
+    except Exception as exp:
         return_response["status"] = "failed"
         return_response["error"] = str(exp)
         notification_message = "Failed to request container association"

cli/medperf/web_ui/datasets/routes.py

@@ -19,7 +19,6 @@
 from medperf.entities.dataset import Dataset
 from medperf.entities.benchmark import Benchmark
 from medperf.entities.execution import Execution
-from medperf.exceptions import MedperfException
 from medperf.web_ui.common import (
     templates,
     check_user_ui,
@@ -190,7 +189,7 @@ def register_dataset(
         return_response["status"] = "success"
         return_response["dataset_id"] = dataset_id
         notification_message = "Dataset successfully registered"
-    except MedperfException as exp:
+    except Exception as exp:
         return_response["status"] = "failed"
         return_response["error"] = str(exp)
         notification_message = "Failed to register dataset"
@@ -225,7 +224,7 @@ def prepare(
         return_response["status"] = "success"
         return_response["dataset_id"] = dataset_id
         notification_message = "Dataset successfully prepared"
-    except MedperfException as exp:
+    except Exception as exp:
         return_response["status"] = "failed"
         return_response["error"] = str(exp)
         notification_message = "Failed to prepare dataset"
@@ -256,7 +255,7 @@ def set_operational(
         return_response["status"] = "success"
         return_response["dataset_id"] = dataset_id
         notification_message = "Dataset successfully set to operational"
-    except MedperfException as exp:
+    except Exception as exp:
         return_response["status"] = "failed"
         return_response["error"] = str(exp)
         notification_message = "Failed to set dataset to operational"
@@ -287,7 +286,7 @@ def associate(
         AssociateDataset.run(data_uid=dataset_id, benchmark_uid=benchmark_id)
         return_response["status"] = "success"
         notification_message = "Successfully requested dataset association"
-    except MedperfException as exp:
+    except Exception as exp:
         return_response["status"] = "failed"
         return_response["error"] = str(exp)
         notification_message = "Failed to request dataset association"
@@ -326,7 +325,7 @@ def run(
         )
         return_response["status"] = "success"
         notification_message = "Execution ran successfully"
-    except MedperfException as exp:
+    except Exception as exp:
         return_response["status"] = "failed"
         return_response["error"] = str(exp)
         notification_message = "Error during execution"
@@ -356,7 +355,7 @@ def submit_result(
         ResultSubmission.run(result_id)
         return_response["status"] = "success"
         notification_message = "Result successfully submitted"
-    except MedperfException as exp:
+    except Exception as exp:
         return_response["status"] = "failed"
         return_response["error"] = str(exp)
         notification_message = "Failed to submit results"
@@ -412,7 +411,7 @@ def export_dataset(
         ExportDataset.run(dataset_id, output_path)
         return_response["status"] = "success"
         notification_message = "Dataset successfully exported"
-    except MedperfException as exp:
+    except Exception as exp:
         return_response["status"] = "failed"
         return_response["error"] = str(exp)
         notification_message = "Failed to export dataset"
@@ -457,7 +456,7 @@ def import_dataset(
         ImportDataset.run(dataset_id, input_path, raw_dataset_path)
         return_response["status"] = "success"
         notification_message = "Dataset successfully imported"
-    except MedperfException as exp:
+    except Exception as exp:
         dataset_id = None
         return_response["status"] = "failed"
         return_response["error"] = str(exp)

cli/medperf/web_ui/medperf_login.py

@@ -1,7 +1,6 @@
 from fastapi import Request, Form, APIRouter, Depends
 from fastapi.responses import HTMLResponse, JSONResponse
 
-from medperf.exceptions import MedperfException
 from medperf.web_ui.common import (
     add_notification,
     initialize_state_task,
@@ -71,7 +70,7 @@ def login(
             templates.env.globals["logged_in"] = True
             return_response["status"] = "success"
             notification_message = "Successfully Logged In"
-        except MedperfException as exp:
+        except Exception as exp:
             return_response["status"] = "failed"
             return_response["error"] = str(exp)
             notification_message = "Error Logging In"
@@ -96,6 +95,6 @@ def logout(
         config.auth.logout()
         templates.env.globals["logged_in"] = False
         return {"status": "success", "error": ""}
-    except MedperfException as e:
+    except Exception as e:
         logger.exception(e)
         return {"status": "failed", "error": "Logout failed. Check logs."}