feat(DIST-S1): support using dataset_util on mozart (i.e. outside of verdi)

Author: chrisjrd

dist_s1/dataset_util.py

@@ -8,7 +8,7 @@
 from typing import Union
 
 from util.datasets_json_util import DatasetsJson
-
+from util.job_util import is_running_outside_verdi_worker_context
 
 logger = logging.getLogger(__name__)
 
@@ -42,15 +42,21 @@ def make_dataset_dir(dataset_id, dataset_type: str = None):
 
 
 def validate_dataset_type(dataset_type):
-    datasets_json = DatasetsJson(file="datasets.json")
+    if is_running_outside_verdi_worker_context():
+        datasets_json = DatasetsJson(file="datasets.json")
+    else:
+        datasets_json = DatasetsJson()
     try:
         datasets_json.get(dataset_type)
     except KeyError as e:
         raise Exception(f"Invalid {dataset_type=}. Compare against datasets.json") from e
 
 
 def is_valid_dataset_type(dataset_type = None):
-    datasets_json = DatasetsJson(file="datasets.json")
+    if is_running_outside_verdi_worker_context():
+        datasets_json = DatasetsJson(file="datasets.json")
+    else:
+        datasets_json = DatasetsJson()
     try:
         datasets_json.get(dataset_type)
         return True