Skip to content

Commit 48c875f

Browse files
authored
[None][fix] Add OpenSearch URL in slurm_launch.sh for Multinode Perf Sanity Test (#9990)
Signed-off-by: Chenfei Zhang <chenfeiz@nvidia.com>
1 parent cc1323b commit 48c875f

File tree

2 files changed

+32
-1
lines changed

2 files changed

+32
-1
lines changed

jenkins/L0_Test.groovy

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1058,12 +1058,31 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
10581058
""".replaceAll("(?m)^\\s*", "")
10591059
}
10601060

1061+
// Define environment variables to export
1062+
def envVarNames = [
1063+
'OPEN_SEARCH_DB_BASE_URL',
1064+
'OPEN_SEARCH_DB_CREDENTIALS',
1065+
'BUILD_ID',
1066+
'BUILD_URL',
1067+
'JOB_NAME',
1068+
'globalVars',
1069+
'gitlabCommit'
1070+
]
1071+
def envVarsToExport = [:]
1072+
envVarNames.each { varName ->
1073+
envVarsToExport[varName] = env."${varName}"
1074+
}
1075+
10611076
srunArgs = [
1077+
"--container-name=multi_node_test-\${SLURM_JOB_ID}",
10621078
"--container-image=$containerImageArg",
10631079
"--container-workdir=/home/svc_tensorrt/bloom/scripts",
10641080
"--container-mounts=$mounts",
10651081
"--container-env=NVIDIA_IMEX_CHANNELS"
10661082
]
1083+
envVarsToExport.each { varName, varValue ->
1084+
srunArgs.add("--container-env=${varName}")
1085+
}
10671086
if(nodeCount > 1) {
10681087
srunArgs.add("--mpi=pmi2")
10691088
}
@@ -1072,6 +1091,17 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
10721091
if (cluster.host.contains("oci-nrt") || cluster.host.contains("oci-hsg") || cluster.host.contains("lbd-lax")) {
10731092
exemptionComment = """--comment='{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"90","reason":"other","description":"Long data and model loading time and disaggregated serving tests"}}'"""
10741093
}
1094+
1095+
def envExportStatements = envVarsToExport.collect { varName, varValue ->
1096+
def escapedValue = varValue?.toString() ?: ''
1097+
escapedValue = escapedValue
1098+
.replace('\\', '\\\\') // Backslash
1099+
.replace('"', '\\"') // Double quote
1100+
.replace('$', '\\$') // Dollar sign (prevent variable expansion)
1101+
.replace('`', '\\`') // Backtick (prevent command substitution)
1102+
"export ${varName}=\"${escapedValue}\""
1103+
}.join('\n')
1104+
10751105
def scriptContent = """#!/bin/bash
10761106
#SBATCH ${exemptionComment} --output=${outputPath}
10771107
${taskArgs.collect { "#SBATCH $it" }.join('\n')}
@@ -1092,6 +1122,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
10921122
export coverageConfigFile="$coverageConfigFile"
10931123
export NVIDIA_IMEX_CHANNELS=\${NVIDIA_IMEX_CHANNELS:-0}
10941124
export NVIDIA_VISIBLE_DEVICES=\${NVIDIA_VISIBLE_DEVICES:-\$(seq -s, 0 \$((\$(nvidia-smi --query-gpu=count -i 0 --format=noheader)-1)))}
1125+
${envExportStatements}
10951126
10961127
echo "Env NVIDIA_IMEX_CHANNELS: \$NVIDIA_IMEX_CHANNELS"
10971128
echo "Env NVIDIA_VISIBLE_DEVICES: \$NVIDIA_VISIBLE_DEVICES"

tests/integration/defs/perf/open_search_db_utils.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ def get_job_info():
145145

146146
# Set trigger_mr_commit to commit
147147
trigger_mr_commit = commit
148-
artifact_url = f"https://urm.nvidia.com/artifactory/sw-tensorrt-generic/llm-artifacts/LLM/main/L0_PostMerge/{job_id}" if job_id else ""
148+
artifact_url = f"https://urm.nvidia.com/artifactory/sw-tensorrt-generic/llm-artifacts/LLM/main/L0_MergeRequest_PR/{job_id}" if job_id else ""
149149
else:
150150
artifact_url = f"https://urm.nvidia.com/artifactory/sw-tensorrt-generic/llm-artifacts/LLM/main/L0_PostMerge/{job_id}" if job_id else ""
151151

0 commit comments

Comments
 (0)