Skip to content

Commit 73b2a4b

Browse files
neomatamuneneomatamune
committed
Use RunTemplate.orchestratorType to build new workflow
1 parent 74e8fc2 commit 73b2a4b

File tree

1 file changed

+179
-88
lines changed

1 file changed

+179
-88
lines changed

‎scenariorun/src/main/kotlin/com/cosmotech/scenariorun/ContainerFactory.kt‎

Lines changed: 179 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import com.cosmotech.scenariorun.domain.ScenarioRunStartContainers
3333
import com.cosmotech.solution.api.SolutionApiService
3434
import com.cosmotech.solution.domain.RunTemplate
3535
import com.cosmotech.solution.domain.RunTemplateHandlerId
36+
import com.cosmotech.solution.domain.RunTemplateOrchestrator
3637
import com.cosmotech.solution.domain.RunTemplateStepSource
3738
import com.cosmotech.solution.domain.Solution
3839
import com.cosmotech.solution.utils.getCloudPath
@@ -60,6 +61,7 @@ private const val CONTAINER_RUN = "runContainer"
6061
private const val CONTAINER_RUN_MODE = "engine"
6162
private const val CONTAINER_POSTRUN = "postRunContainer"
6263
private const val CONTAINER_POSTRUN_MODE = "postrun"
64+
private const val CONTAINER_CSM_ORC = "CSMOrchestrator"
6365
internal const val IDENTITY_PROVIDER = "IDENTITY_PROVIDER"
6466
internal const val AZURE_TENANT_ID_VAR = "AZURE_TENANT_ID"
6567
internal const val AZURE_CLIENT_ID_VAR = "AZURE_CLIENT_ID"
@@ -94,7 +96,9 @@ private const val AZURE_DATA_EXPLORER_RESOURCE_INGEST_URI_VAR =
9496
private const val AZURE_DATA_EXPLORER_DATABASE_NAME = "AZURE_DATA_EXPLORER_DATABASE_NAME"
9597
private const val RUN_TEMPLATE_ID_VAR = "CSM_RUN_TEMPLATE_ID"
9698
private const val CONTAINER_MODE_VAR = "CSM_CONTAINER_MODE"
99+
private const val CONTAINER_ORCHESTRATOR_LEGACY_VAR = "CSM_ENTRYPOINT_LEGACY"
97100
private const val ENTRYPOINT_NAME = "entrypoint.py"
101+
private const val ENTRYPOINT_NAME_CSM_ORC = "csm-orc"
98102
private const val EVENT_HUB_MEASURES_VAR = "CSM_PROBES_MEASURES_TOPIC"
99103
internal const val EVENT_HUB_CONTROL_PLANE_VAR = "CSM_CONTROL_PLANE_TOPIC"
100104
private const val CSM_SIMULATION_VAR = "CSM_SIMULATION"
@@ -137,6 +141,8 @@ private val LABEL_SIZING =
137141
NODE_LABEL_HIGH_MEMORY to HIGH_MEMORY_SIZING,
138142
)
139143

144+
private val CSM_ORC_ORCHESTRATOR_VALUE = RunTemplateOrchestrator.csmMinusOrc.value
145+
140146
@Component
141147
@Suppress("LargeClass", "TooManyFunctions")
142148
class ContainerFactory(
@@ -408,6 +414,10 @@ class ContainerFactory(
408414
))
409415
}
410416

417+
internal fun getOrchestratorType(orchestratorType: String?): String {
418+
return orchestratorType ?: CSM_ORC_ORCHESTRATOR_VALUE
419+
}
420+
411421
@Suppress("LongMethod", "LongParameterList") // Exception for this method - too tedious to update
412422
internal fun buildContainersPipeline(
413423
scenario: Scenario,
@@ -430,145 +440,225 @@ class ContainerFactory(
430440

431441
var containers: MutableList<ScenarioRunContainer> = mutableListOf()
432442

433-
var currentDependencies: MutableList<String>? = mutableListOf()
434-
435-
containers.addAll(
436-
buildFetchDatasetsContainersPipeline(
437-
currentDependencies,
438-
template,
439-
datasets,
440-
connectors,
441-
scenario,
442-
organization,
443-
workspace,
444-
csmSimulationId,
445-
NODE_LABEL_DEFAULT,
446-
BASIC_SIZING))
447-
448-
if (scenarioDataDownload) {
443+
if (getOrchestratorType(template.orchestratorType?.value) == CSM_ORC_ORCHESTRATOR_VALUE) {
449444
containers.addAll(
450-
buildScenarioDataDownloadContainersPipeline(
451-
currentDependencies,
445+
buildCSMOrchestratorPipeline(
452446
organization,
453447
workspace,
454448
scenario,
455449
solution,
456-
template,
450+
runTemplateId,
457451
csmSimulationId,
458-
scenarioDataDownloadJobId!!,
459-
NODE_LABEL_DEFAULT,
460-
BASIC_SIZING))
452+
scenarioRunLabel,
453+
scenarioRunSizing))
461454
} else {
455+
var currentDependencies: MutableList<String>? = mutableListOf()
456+
462457
containers.addAll(
463-
buildFetchScenarioParametersContainersPipeline(
458+
buildFetchDatasetsContainersPipeline(
464459
currentDependencies,
465460
template,
461+
datasets,
462+
connectors,
463+
scenario,
466464
organization,
467465
workspace,
468-
scenario,
469466
csmSimulationId,
470-
solution,
471-
datasets,
472-
connectors,
473467
NODE_LABEL_DEFAULT,
474468
BASIC_SIZING))
475469

476-
if (currentDependencies.isNullOrEmpty()) {
477-
currentDependencies = null
478-
}
479-
480-
if (testStep(template.applyParameters)) {
470+
if (scenarioDataDownload) {
481471
containers.addAll(
482-
buildApplyParametersContainersPipeline(
472+
buildScenarioDataDownloadContainersPipeline(
483473
currentDependencies,
484474
organization,
485475
workspace,
486476
scenario,
487477
solution,
488-
runTemplateId,
478+
template,
489479
csmSimulationId,
480+
scenarioDataDownloadJobId!!,
490481
NODE_LABEL_DEFAULT,
491482
BASIC_SIZING))
492-
currentDependencies = mutableListOf(CONTAINER_APPLY_PARAMETERS)
493-
}
494-
if (testStep(template.validateData)) {
483+
} else {
495484
containers.addAll(
496-
buildValidateDataContainersPipeline(
485+
buildFetchScenarioParametersContainersPipeline(
497486
currentDependencies,
487+
template,
498488
organization,
499489
workspace,
500490
scenario,
501-
solution,
502-
runTemplateId,
503491
csmSimulationId,
492+
solution,
493+
datasets,
494+
connectors,
504495
NODE_LABEL_DEFAULT,
505496
BASIC_SIZING))
506-
currentDependencies = mutableListOf(CONTAINER_VALIDATE_DATA)
507-
}
508497

509-
containers.addAll(
510-
buildSendDataWarehouseContainersPipeline(
511-
currentDependencies,
512-
workspace,
513-
template,
514-
organization,
515-
scenario,
516-
csmSimulationId,
517-
NODE_LABEL_DEFAULT,
518-
BASIC_SIZING))
498+
if (currentDependencies.isNullOrEmpty()) {
499+
currentDependencies = null
500+
}
519501

520-
if (testStep(template.preRun)) {
521-
containers.addAll(
522-
buildPreRunContainersPipeline(
523-
currentDependencies,
524-
organization,
525-
workspace,
526-
scenario,
527-
solution,
528-
runTemplateId,
529-
csmSimulationId,
530-
NODE_LABEL_DEFAULT,
531-
BASIC_SIZING))
532-
currentDependencies = mutableListOf(CONTAINER_PRERUN)
533-
}
502+
if (testStep(template.applyParameters)) {
503+
containers.addAll(
504+
buildApplyParametersContainersPipeline(
505+
currentDependencies,
506+
organization,
507+
workspace,
508+
scenario,
509+
solution,
510+
runTemplateId,
511+
csmSimulationId,
512+
NODE_LABEL_DEFAULT,
513+
BASIC_SIZING))
514+
currentDependencies = mutableListOf(CONTAINER_APPLY_PARAMETERS)
515+
}
516+
if (testStep(template.validateData)) {
517+
containers.addAll(
518+
buildValidateDataContainersPipeline(
519+
currentDependencies,
520+
organization,
521+
workspace,
522+
scenario,
523+
solution,
524+
runTemplateId,
525+
csmSimulationId,
526+
NODE_LABEL_DEFAULT,
527+
BASIC_SIZING))
528+
currentDependencies = mutableListOf(CONTAINER_VALIDATE_DATA)
529+
}
534530

535-
if (testStep(template.run)) {
536531
containers.addAll(
537-
buildRunContainersPipeline(
532+
buildSendDataWarehouseContainersPipeline(
538533
currentDependencies,
539-
organization,
540534
workspace,
541-
scenario,
542-
solution,
543-
runTemplateId,
544-
csmSimulationId,
545-
scenarioRunLabel,
546-
scenarioRunSizing))
547-
currentDependencies = mutableListOf(CONTAINER_RUN)
548-
}
549-
550-
if (testStep(template.postRun)) {
551-
containers.addAll(
552-
buildPostRunContainersPipeline(
553-
currentDependencies,
535+
template,
554536
organization,
555-
workspace,
556537
scenario,
557-
solution,
558-
runTemplateId,
559538
csmSimulationId,
560539
NODE_LABEL_DEFAULT,
561540
BASIC_SIZING))
541+
542+
if (testStep(template.preRun)) {
543+
containers.addAll(
544+
buildPreRunContainersPipeline(
545+
currentDependencies,
546+
organization,
547+
workspace,
548+
scenario,
549+
solution,
550+
runTemplateId,
551+
csmSimulationId,
552+
NODE_LABEL_DEFAULT,
553+
BASIC_SIZING))
554+
currentDependencies = mutableListOf(CONTAINER_PRERUN)
555+
}
556+
557+
if (testStep(template.run)) {
558+
containers.addAll(
559+
buildRunContainersPipeline(
560+
currentDependencies,
561+
organization,
562+
workspace,
563+
scenario,
564+
solution,
565+
runTemplateId,
566+
csmSimulationId,
567+
scenarioRunLabel,
568+
scenarioRunSizing))
569+
currentDependencies = mutableListOf(CONTAINER_RUN)
570+
}
571+
572+
if (testStep(template.postRun)) {
573+
containers.addAll(
574+
buildPostRunContainersPipeline(
575+
currentDependencies,
576+
organization,
577+
workspace,
578+
scenario,
579+
solution,
580+
runTemplateId,
581+
csmSimulationId,
582+
NODE_LABEL_DEFAULT,
583+
BASIC_SIZING))
584+
}
562585
}
563-
}
564586

565-
if (template.stackSteps == true) {
566-
containers = stackSolutionContainers(containers)
587+
if (template.stackSteps == true) {
588+
containers = stackSolutionContainers(containers)
589+
}
567590
}
568591

569592
return containers.toList()
570593
}
571594

595+
private fun buildCSMOrchestratorPipeline(
596+
organization: Organization,
597+
workspace: Workspace,
598+
scenario: Scenario,
599+
solution: Solution,
600+
runTemplateId: String,
601+
csmSimulationId: String,
602+
nodeSizingLabel: String,
603+
customSizing: Sizing
604+
) =
605+
listOf(
606+
this.buildCSMOrchestratorContainer(
607+
organization,
608+
workspace,
609+
scenario,
610+
solution,
611+
runTemplateId,
612+
csmSimulationId,
613+
nodeSizingLabel,
614+
customSizing))
615+
616+
internal fun buildCSMOrchestratorContainer(
617+
organization: Organization,
618+
workspace: Workspace,
619+
scenario: Scenario,
620+
solution: Solution,
621+
runTemplateId: String,
622+
csmSimulationId: String,
623+
nodeSizingLabel: String,
624+
customSizing: Sizing
625+
): ScenarioRunContainer {
626+
627+
val imageName =
628+
getImageName(
629+
csmPlatformProperties.azure?.containerRegistries?.solutions ?: "",
630+
solution.repository,
631+
solution.version)
632+
val envVars =
633+
getCommonEnvVars(
634+
csmPlatformProperties,
635+
csmSimulationId,
636+
organization.id ?: "",
637+
workspace.id ?: "",
638+
scenario.id ?: "",
639+
workspace.key)
640+
envVars[RUN_TEMPLATE_ID_VAR] = runTemplateId
641+
envVars[CONTAINER_MODE_VAR] = CSM_ORC_ORCHESTRATOR_VALUE
642+
envVars[CONTAINER_ORCHESTRATOR_LEGACY_VAR] = "false"
643+
644+
envVars.putAll(getEventHubEnvVars(organization, workspace))
645+
646+
val template = getRunTemplate(solution, runTemplateId)
647+
val csmSimulation = template.csmSimulation
648+
if (csmSimulation != null) {
649+
envVars[CSM_SIMULATION_VAR] = csmSimulation
650+
}
651+
return ScenarioRunContainer(
652+
name = CONTAINER_CSM_ORC,
653+
image = imageName,
654+
envVars = envVars,
655+
dependencies = null,
656+
entrypoint = ENTRYPOINT_NAME_CSM_ORC,
657+
solutionContainer = true,
658+
nodeLabel = nodeSizingLabel,
659+
runSizing = customSizing.toContainerResourceSizing())
660+
}
661+
572662
private fun buildPostRunContainersPipeline(
573663
currentDependencies: MutableList<String>?,
574664
organization: Organization,
@@ -1260,6 +1350,7 @@ class ContainerFactory(
12601350
workspace.key)
12611351
envVars[RUN_TEMPLATE_ID_VAR] = runTemplateId
12621352
envVars[CONTAINER_MODE_VAR] = step.mode
1353+
envVars[CONTAINER_ORCHESTRATOR_LEGACY_VAR] = "true"
12631354

12641355
envVars.putAll(getEventHubEnvVars(organization, workspace))
12651356

0 commit comments

Comments
 (0)