diff --git a/CosmoTech_Acceleration_Library/Accelerators/scenario_download/scenario_downloader.py b/CosmoTech_Acceleration_Library/Accelerators/scenario_download/scenario_downloader.py index 08592617..0d6d520f 100644 --- a/CosmoTech_Acceleration_Library/Accelerators/scenario_download/scenario_downloader.py +++ b/CosmoTech_Acceleration_Library/Accelerators/scenario_download/scenario_downloader.py @@ -113,6 +113,7 @@ def download_dataset(self, dataset_id: str) -> (str, str, Union[str, None]): is_adt = 'AZURE_DIGITAL_TWINS_URL' in parameters is_storage = 'AZURE_STORAGE_CONTAINER_BLOB_PREFIX' in parameters is_legacy_twin_cache = 'TWIN_CACHE_NAME' in parameters and dataset.twingraph_id is None # Legacy twingraph dataset with specific connector + is_in_workspace_file = 'workspaceFile' in dataset.tags if is_adt: return { @@ -137,6 +138,16 @@ def download_dataset(self, dataset_id: str) -> (str, str, Union[str, None]): "content": _content, "name": dataset.name } + elif is_in_workspace_file: + _file_name = dataset.source.location + _content = self._download_file(_file_name) + self.dataset_file_temp_path[dataset_id] = self.dataset_file_temp_path[_file_name] + return { + "type": _file_name.split('.')[-1], + "content": _content, + "name": dataset.name + } + else: return { "type": "twincache", diff --git a/CosmoTech_Acceleration_Library/__init__.py b/CosmoTech_Acceleration_Library/__init__.py index e796c95f..540fdb98 100644 --- a/CosmoTech_Acceleration_Library/__init__.py +++ b/CosmoTech_Acceleration_Library/__init__.py @@ -1,4 +1,4 @@ # Copyright (c) Cosmo Tech corporation. # Licensed under the MIT license. -__version__ = '0.8.4' +__version__ = '0.8.5' diff --git a/cosmotech/coal/cli/commands/api/run_load_data.py b/cosmotech/coal/cli/commands/api/run_load_data.py index 2bd74d3a..53dcf643 100644 --- a/cosmotech/coal/cli/commands/api/run_load_data.py +++ b/cosmotech/coal/cli/commands/api/run_load_data.py @@ -10,7 +10,6 @@ import pathlib from shutil import copytree -from cosmotech_api.api.dataset_api import DatasetApi from cosmotech_api.api.runner_api import RunnerApi from cosmotech_api.api.workspace_api import WorkspaceApi @@ -35,11 +34,16 @@ def download_runner_data(organization_id: str, workspace_id: str, runner_id: str _dl = ScenarioDownloader(workspace_id=workspace_id, organization_id=organization_id, read_files=False) with get_api_client()[0] as api_client: runner_api_instance = RunnerApi(api_client) - dataset_api_instance = DatasetApi(api_client) workspace_api_instance = WorkspaceApi(api_client) runner_data = runner_api_instance.get_runner(organization_id=organization_id, workspace_id=workspace_id, runner_id=runner_id) + + # skip if no parameters found + if not runner_data.parameters_values: + LOGGER.warning('no parameters found in the runner') + return + LOGGER.info("Loaded run data") # Pre-read of all workspace files to ensure ready to download AZ storage files all_api_files = workspace_api_instance.find_all_workspace_files(