fix: empty datasets should not be downloaded

Author: neomatamune

CosmoTech_Acceleration_Library/Accelerators/scenario_download/scenario_downloader.py

@@ -8,15 +8,14 @@
 import tempfile
 from typing import Union
 
-import cosmotech_api
 from azure.digitaltwins.core import DigitalTwinsClient
 from azure.identity import DefaultAzureCredential
 from cosmotech_api import DatasetApi
+from cosmotech_api import DatasetTwinGraphQuery
 from cosmotech_api import ScenarioApi
+from cosmotech_api import TwinGraphQuery
 from cosmotech_api import TwingraphApi
 from cosmotech_api import WorkspaceApi
-from cosmotech_api import DatasetTwinGraphQuery
-from cosmotech_api import TwinGraphQuery
 from openpyxl import load_workbook
 
 from cosmotech.coal.cosmotech_api.connection import get_api_client
@@ -85,7 +84,6 @@ def __init__(
             self.credentials = DefaultAzureCredential()
         else:
             self.credentials = None
-        
 
         self.workspace_id = workspace_id
         self.organization_id = organization_id
@@ -104,19 +102,17 @@ def get_scenario_data(self, scenario_id: str):
     def download_dataset(self, dataset_id: str) -> (str, str, Union[str, None]):
         with get_api_client()[0] as api_client:
             api_instance = DatasetApi(api_client)
-
             dataset = api_instance.find_dataset_by_id(
                 organization_id=self.organization_id,
                 dataset_id=dataset_id)
             if dataset.connector is None:
                 parameters = []
             else:
                 parameters = dataset.connector.parameters_values
-
             is_adt = 'AZURE_DIGITAL_TWINS_URL' in parameters
             is_storage = 'AZURE_STORAGE_CONTAINER_BLOB_PREFIX' in parameters
             is_legacy_twin_cache = 'TWIN_CACHE_NAME' in parameters and dataset.twingraph_id is None  # Legacy twingraph dataset with specific connector
-            is_in_workspace_file = 'workspaceFile' in dataset.tags
+            is_in_workspace_file = False if dataset.tags is None else 'workspaceFile' in dataset.tags
 
             if is_adt:
                 return {
@@ -329,7 +325,7 @@ def get_all_datasets(self, scenario_id: str) -> dict:
         dataset_ids = datasets[:]
 
         for parameter in scenario_data.parameters_values:
-            if parameter.var_type == '%DATASETID%':
+            if parameter.var_type == '%DATASETID%' and parameter.value:
                 dataset_id = parameter.value
                 dataset_ids.append(dataset_id)
 
@@ -344,7 +340,7 @@ def download_dataset_process(_dataset_id, _return_dict, _error_dict):
                 _error_dict[_dataset_id] = f'{type(e).__name__}: {str(e)}'
                 raise e
 
-        if self.parallel:
+        if self.parallel and len(dataset_ids) > 1:
             manager = multiprocessing.Manager()
             return_dict = manager.dict()
             error_dict = manager.dict()

CosmoTech_Acceleration_Library/__init__.py

@@ -1,4 +1,4 @@
 # Copyright (c) Cosmo Tech corporation.
 # Licensed under the MIT license.
 
-__version__ = '0.8.7'
+__version__ = '0.8.8'