@@ -1058,12 +1058,31 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
10581058 """ . replaceAll(" (?m)^\\ s*" , " " )
10591059 }
10601060
1061+ // Define environment variables to export
1062+ def envVarNames = [
1063+ ' OPEN_SEARCH_DB_BASE_URL' ,
1064+ ' OPEN_SEARCH_DB_CREDENTIALS' ,
1065+ ' BUILD_ID' ,
1066+ ' BUILD_URL' ,
1067+ ' JOB_NAME' ,
1068+ ' globalVars' ,
1069+ ' gitlabCommit'
1070+ ]
1071+ def envVarsToExport = [:]
1072+ envVarNames. each { varName ->
1073+ envVarsToExport[varName] = env. " ${ varName} "
1074+ }
1075+
10611076 srunArgs = [
1077+ " --container-name=multi_node_test-\$ {SLURM_JOB_ID}" ,
10621078 " --container-image=$containerImageArg " ,
10631079 " --container-workdir=/home/svc_tensorrt/bloom/scripts" ,
10641080 " --container-mounts=$mounts " ,
10651081 " --container-env=NVIDIA_IMEX_CHANNELS"
10661082 ]
1083+ envVarsToExport. each { varName , varValue ->
1084+ srunArgs. add(" --container-env=${ varName} " )
1085+ }
10671086 if (nodeCount > 1 ) {
10681087 srunArgs. add(" --mpi=pmi2" )
10691088 }
@@ -1072,6 +1091,17 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
10721091 if (cluster. host. contains(" oci-nrt" ) || cluster. host. contains(" oci-hsg" ) || cluster. host. contains(" lbd-lax" )) {
10731092 exemptionComment = """ --comment='{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"90","reason":"other","description":"Long data and model loading time and disaggregated serving tests"}}'"""
10741093 }
1094+
1095+ def envExportStatements = envVarsToExport. collect { varName , varValue ->
1096+ def escapedValue = varValue?. toString() ?: ' '
1097+ escapedValue = escapedValue
1098+ .replace(' \\ ' , ' \\\\ ' ) // Backslash
1099+ .replace(' "' , ' \\ "' ) // Double quote
1100+ .replace(' $' , ' \\ $' ) // Dollar sign (prevent variable expansion)
1101+ .replace(' `' , ' \\ `' ) // Backtick (prevent command substitution)
1102+ " export ${ varName} =\" ${ escapedValue} \" "
1103+ }. join(' \n ' )
1104+
10751105 def scriptContent = """ #!/bin/bash
10761106 #SBATCH ${ exemptionComment} --output=${ outputPath}
10771107 ${ taskArgs.collect { "#SBATCH $it" }.join('\n')}
@@ -1092,6 +1122,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
10921122 export coverageConfigFile="$coverageConfigFile "
10931123 export NVIDIA_IMEX_CHANNELS=\$ {NVIDIA_IMEX_CHANNELS:-0}
10941124 export NVIDIA_VISIBLE_DEVICES=\$ {NVIDIA_VISIBLE_DEVICES:-\$ (seq -s, 0 \$ ((\$ (nvidia-smi --query-gpu=count -i 0 --format=noheader)-1)))}
1125+ ${ envExportStatements}
10951126
10961127 echo "Env NVIDIA_IMEX_CHANNELS: \$ NVIDIA_IMEX_CHANNELS"
10971128 echo "Env NVIDIA_VISIBLE_DEVICES: \$ NVIDIA_VISIBLE_DEVICES"
0 commit comments