multiple DATASETID values workflow code & tests

vcarluer · vcarluer · commit 9a2f930669e6 · 2021-11-18T18:00:04.000+01:00
diff --git a/doc/Models/RunTemplateParameter.md b/doc/Models/RunTemplateParameter.md
@@ -11,7 +11,6 @@ Name | Type | Description | Notes
 **maxValue** | **String** | the maximum value for this parameter | [optional] [default to null]
 **regexValidation** | **String** | a regex to validate the value | [optional] [default to null]
 **options** | **Map** | freeform options | [optional] [default to null]
-**isArray** | **Boolean** | whether or not the parameter contains an array of values | [optional] [default to null]
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
 
diff --git a/doc/Models/ScenarioRunTemplateParameterValue.md b/doc/Models/ScenarioRunTemplateParameterValue.md
@@ -5,7 +5,6 @@ Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **parameterId** | **String** | the parameter Id | [default to null]
 **varType** | **String** | the parameter value type | [optional] [default to null]
-**isArray** | **Boolean** | whether or not the parameter contains an array of values | [optional] [default to null]
 **value** | **String** | the parameter value | [default to null]
 **isInherited** | **Boolean** | whether or not the value is inherited from parent or has been changed | [optional] [default to null]
 
diff --git a/openapi/plantuml/schemas.plantuml b/openapi/plantuml/schemas.plantuml
@@ -136,7 +136,6 @@ entity RunTemplateParameter {
     maxValue: String
     regexValidation: String
     options: Map
-    isArray: Boolean
 }
 
 entity RunTemplateParameterGroup {
@@ -315,7 +314,6 @@ entity ScenarioRunStatusNode {
 entity ScenarioRunTemplateParameterValue {
     * parameterId: String
     varType: String
-    isArray: Boolean
     * value: String
     isInherited: Boolean
 }
diff --git a/scenario/src/main/openapi/scenarios.yaml b/scenario/src/main/openapi/scenarios.yaml
@@ -648,10 +648,6 @@ components:
           type: string
           readOnly: true
           description: the parameter value type
-        isArray:
-          type: boolean
-          readOnly: true
-          description: whether or not the parameter contains an array of values
         value:
           type: string
           description: the parameter value
diff --git a/scenariorun/src/main/kotlin/com/cosmotech/scenariorun/ContainerFactory.kt b/scenariorun/src/main/kotlin/com/cosmotech/scenariorun/ContainerFactory.kt
@@ -21,6 +21,7 @@ import com.cosmotech.scenariorun.container.StartInfo
 import com.cosmotech.scenariorun.dataset.PARAMETERS_DATASET_ID
 import com.cosmotech.scenariorun.dataset.findDatasetsAndConnectors
 import com.cosmotech.scenariorun.dataset.getDatasetEnvVars
+import com.cosmotech.scenariorun.dataset.getDatasetIdListFromValue
 import com.cosmotech.scenariorun.domain.ScenarioRunContainer
 import com.cosmotech.scenariorun.domain.ScenarioRunContainerArtifact
 import com.cosmotech.scenariorun.domain.ScenarioRunStartContainers
@@ -750,27 +751,32 @@ internal class ContainerFactory(
                 ?: throw IllegalStateException(
                     "Parameter ${parameterValue.parameterId} not found in Solution ${solution.id}")
         if (parameter.varType == PARAMETERS_DATASET_ID) {
-          val dataset =
-              datasets?.find { dataset -> dataset.id == parameterValue.value }
-                  ?: throw IllegalStateException(
-                      "Dataset ${parameterValue.value} cannot be found in Datasets")
-          val connector =
-              connectors?.find { connector -> connector.id == dataset.connector?.id }
-                  ?: throw IllegalStateException(
-                      "Connector id ${dataset.connector?.id} not found in connectors list")
-
-          containers.add(
-              buildFromDataset(
-                  dataset,
-                  connector,
-                  datasetParameterCount,
-                  true,
-                  parameter.id,
-                  organizationId,
-                  workspaceId,
-                  workspaceKey,
-                  csmSimulationId))
-          datasetParameterCount++
+          val datasetIdList = getDatasetIdListFromValue(parameterValue.value)
+          datasetIdList.forEachIndexed { index, datasetId ->
+            val dataset =
+                datasets?.find { dataset -> dataset.id == datasetId }
+                    ?: throw IllegalStateException(
+                        "Dataset ${datasetId} cannot be found in Datasets")
+            val connector =
+                connectors?.find { connector -> connector.id == dataset.connector?.id }
+                    ?: throw IllegalStateException(
+                        "Connector id ${dataset.connector?.id} not found in connectors list")
+
+            val fetchId =
+                if (datasetIdList.size == 1) parameter.id else "${parameter.id}-${index.toString()}"
+            containers.add(
+                buildFromDataset(
+                    dataset,
+                    connector,
+                    datasetParameterCount,
+                    true,
+                    fetchId,
+                    organizationId,
+                    workspaceId,
+                    workspaceKey,
+                    csmSimulationId))
+            datasetParameterCount++
+          }
         }
       }
     }
diff --git a/scenariorun/src/main/kotlin/com/cosmotech/scenariorun/dataset/DatasetUtils.kt b/scenariorun/src/main/kotlin/com/cosmotech/scenariorun/dataset/DatasetUtils.kt
@@ -44,12 +44,12 @@ internal fun findDatasetsAndConnectors(
           ?.forEach { parameter ->
             val parameterValue = scenario.parametersValues?.find { it.parameterId == parameter.id }
             addParameterValue(
-              parameterValue,
-              datasetService,
-              connectorService,
-              organizationId,
-              datasets,
-              connectors,
+                parameterValue,
+                datasetService,
+                connectorService,
+                organizationId,
+                datasets,
+                connectors,
             )
           }
     }
@@ -59,39 +59,36 @@ internal fun findDatasetsAndConnectors(
 }
 
 private fun addParameterValue(
-              parameterValue: ScenarioRunTemplateParameterValue?,
-              datasetService: DatasetApiService,
-              connectorService: ConnectorApiService,
-              organizationId: String,
-              datasets: MutableMap<String, Dataset>,
-              connectors: MutableMap<String, Connector>,
-            ) {
-    if (parameterValue != null && parameterValue.value != "") {
-      val value = parameterValue.value
-      if (value.startsWith("[")) {
-        val datasetIds = value.substring(1, value.length - 1)
-        val datasetIdList = datasetIds.split(",")
-        datasetIdList.forEach {
-          addDatasetAndConnector(
-              datasetService,
-              connectorService,
-              organizationId,
-              it,
-              datasets,
-              connectors,
-          )
-        }
-      } else {
-        addDatasetAndConnector(
-            datasetService,
-            connectorService,
-            organizationId,
-            value,
-            datasets,
-            connectors,
-        )
-      }
+    parameterValue: ScenarioRunTemplateParameterValue?,
+    datasetService: DatasetApiService,
+    connectorService: ConnectorApiService,
+    organizationId: String,
+    datasets: MutableMap<String, Dataset>,
+    connectors: MutableMap<String, Connector>,
+) {
+  if (parameterValue != null && parameterValue.value != "") {
+    val value = parameterValue.value
+    val datasetIdList = getDatasetIdListFromValue(value)
+    datasetIdList.forEach {
+      addDatasetAndConnector(
+          datasetService,
+          connectorService,
+          organizationId,
+          it,
+          datasets,
+          connectors,
+      )
     }
+  }
+}
+
+internal fun getDatasetIdListFromValue(paramValue: String): List<String> {
+  if (paramValue.startsWith("[")) {
+    val datasetIds = paramValue.substring(1, paramValue.length - 1)
+    return datasetIds.split(",")
+  } else {
+    return listOf(paramValue)
+  }
 }
 
 internal fun addDatasetAndConnector(
diff --git a/scenariorun/src/test/kotlin/com/cosmotech/scenariorun/ContainerFactoryTests.kt b/scenariorun/src/test/kotlin/com/cosmotech/scenariorun/ContainerFactoryTests.kt
@@ -1377,12 +1377,95 @@ class ContainerFactoryTests {
 
   @Test
   fun `Build all containers for a Scenario DATASETID containers env vars`() {
-    parametersDatasetEnvTest("1", "param2")
+    parametersDatasetEnvTest(getScenarioDatasetIds(), "1", "param2")
   }
 
   @Test
   fun `Build all containers for a Scenario DATASETID containers env vars 2`() {
-    parametersDatasetEnvTest("2", "param3")
+    parametersDatasetEnvTest(getScenarioDatasetIds(), "2", "param3")
+  }
+
+  @Test
+  fun `Build all containers for a Scenario Two DATASETID containers name list`() {
+    val scenario = getScenarioTwoDatasetIds()
+    val datasets = listOf(getDataset(), getDataset2(), getDataset3())
+    val connectors = listOf(getConnector(), getConnector2(), getConnector3())
+    val workspace = getWorkspace()
+    val solution = getSolutionDatasetIds()
+    val containers =
+        factory.buildContainersPipeline(
+            scenario,
+            datasets,
+            connectors,
+            workspace,
+            getOrganization(),
+            solution,
+            CSM_SIMULATION_ID)
+    val expected =
+        listOf(
+            "fetchDatasetContainer-1",
+            "fetchScenarioParametersContainer",
+            "fetchScenarioDatasetParametersContainer-1",
+            "fetchScenarioDatasetParametersContainer-2",
+            "fetchScenarioDatasetParametersContainer-3",
+            "fetchScenarioDatasetParametersContainer-4",
+            "applyParametersContainer",
+            "validateDataContainer",
+            "sendDataWarehouseContainer",
+            "preRunContainer",
+            "runContainer",
+            "postRunContainer",
+        )
+    assertEquals(expected, containers.map { container -> container.name })
+  }
+
+  @Test
+  fun `Build all containers for a Scenario Two DATASETID containers image list`() {
+    val scenario = getScenarioTwoDatasetIds()
+    val datasets = listOf(getDataset(), getDataset2(), getDataset3())
+    val connectors = listOf(getConnector(), getConnector2(), getConnector3())
+    val workspace = getWorkspace()
+    val solution = getSolutionDatasetIds()
+    val containers =
+        factory.buildContainersPipeline(
+            scenario,
+            datasets,
+            connectors,
+            workspace,
+            getOrganization(),
+            solution,
+            CSM_SIMULATION_ID)
+    val expected =
+        listOf(
+            "ghcr.io/cosmotech/test_connector:1.0.0",
+            "ghcr.io/cosmotech/scenariofetchparameters:1.0.0",
+            "ghcr.io/cosmotech/test_connector:1.0.0",
+            "ghcr.io/cosmotech/test_connector2:1.0.0",
+            "ghcr.io/cosmotech/test_connector2:1.0.0",
+            "ghcr.io/cosmotech/test_connector3:1.0.0",
+            "twinengines.azurecr.io/cosmotech/testsolution_simulator:1.0.0",
+            "twinengines.azurecr.io/cosmotech/testsolution_simulator:1.0.0",
+            "ghcr.io/cosmotech/senddatawarehouse:1.0.0",
+            "twinengines.azurecr.io/cosmotech/testsolution_simulator:1.0.0",
+            "twinengines.azurecr.io/cosmotech/testsolution_simulator:1.0.0",
+            "twinengines.azurecr.io/cosmotech/testsolution_simulator:1.0.0",
+        )
+    assertEquals(expected, containers.map { container -> container.image })
+  }
+
+  @Test
+  fun `Build all containers for a Scenario Two DATASETID containers env vars 2-1`() {
+    parametersDatasetEnvTest(getScenarioTwoDatasetIds(), "1", "param2-0")
+  }
+
+  @Test
+  fun `Build all containers for a Scenario Two DATASETID containers env vars 2-2`() {
+    parametersDatasetEnvTest(getScenarioTwoDatasetIds(), "2", "param2-1")
+  }
+
+  @Test
+  fun `Build all containers for a Scenario Two DATASETID containers env vars 3`() {
+    parametersDatasetEnvTest(getScenarioTwoDatasetIds(), "3", "param3")
   }
 
   @Test
@@ -1812,8 +1895,7 @@ class ContainerFactoryTests {
     )
   }
 
-  private fun parametersDatasetEnvTest(nameId: String, param: String) {
-    val scenario = getScenarioDatasetIds()
+  private fun parametersDatasetEnvTest(scenario: Scenario, nameId: String, param: String) {
     val datasets = listOf(getDataset(), getDataset2(), getDataset3())
     val connectors = listOf(getConnector(), getConnector2(), getConnector3())
     val workspace = getWorkspace()
@@ -2243,15 +2325,15 @@ class ContainerFactoryTests {
                 ),
                 RunTemplateParameter(
                     id = "param2",
-                    labels = mapOf("en" to "Parameter Dataset 1"),
+                    labels = mapOf("en" to "Parameter Dataset 2"),
                     varType = "%DATASETID%"),
                 RunTemplateParameter(
                     id = "param3",
-                    labels = mapOf("en" to "Parameter Dataset 2"),
+                    labels = mapOf("en" to "Parameter Dataset 3"),
                     varType = "%DATASETID%"),
                 RunTemplateParameter(
                     id = "param4",
-                    labels = mapOf("en" to "Parameter Dataset 3"),
+                    labels = mapOf("en" to "Parameter Dataset 4"),
                     varType = "%DATASETID%"),
                 RunTemplateParameter(
                     id = "param5",
@@ -2490,6 +2572,37 @@ class ContainerFactoryTests {
             ))
   }
 
+  private fun getScenarioTwoDatasetIds(): Scenario {
+    return Scenario(
+        id = "AQWXSZ",
+        name = "Test Scenario",
+        runTemplateId = "testruntemplate",
+        datasetList = listOf("1"),
+        parametersValues =
+            listOf(
+                ScenarioRunTemplateParameterValue(
+                    parameterId = "param1",
+                    value = "valParam1",
+                ),
+                ScenarioRunTemplateParameterValue(
+                    parameterId = "param2",
+                    value = "[1,2]",
+                ),
+                ScenarioRunTemplateParameterValue(
+                    parameterId = "param3",
+                    value = "2",
+                ),
+                ScenarioRunTemplateParameterValue(
+                    parameterId = "param4",
+                    value = "3",
+                ),
+                ScenarioRunTemplateParameterValue(
+                    parameterId = "param5",
+                    value = "999",
+                ),
+            ))
+  }
+
   private fun getScenarioTwoDatasetsAndDatasetIds(): Scenario {
     return Scenario(
         id = "AQWXSZ",
diff --git a/solution/src/main/openapi/solutions.yaml b/solution/src/main/openapi/solutions.yaml
@@ -682,9 +682,6 @@ components:
           type: object
           description: freeform options
           additionalProperties: true
-        isArray:
-          type: boolean
-          description: whether or not the parameter contains an array of values
       required:
         - id
         - labels

Original file line number	Diff line number	Diff line change
`@@ -136,7 +136,6 @@ entity RunTemplateParameter {`
`136`	`136`	`maxValue: String`
`137`	`137`	`regexValidation: String`
`138`	`138`	`options: Map`
`139`		`- isArray: Boolean`
`140`	`139`	`}`
`141`	`140`
`142`	`141`	`entity RunTemplateParameterGroup {`
`@@ -315,7 +314,6 @@ entity ScenarioRunStatusNode {`
`315`	`314`	`entity ScenarioRunTemplateParameterValue {`
`316`	`315`	`* parameterId: String`
`317`	`316`	`varType: String`
`318`		`- isArray: Boolean`
`319`	`317`	`* value: String`
`320`	`318`	`isInherited: Boolean`
`321`	`319`	`}`