Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 33 additions & 22 deletions main.py
Original file line number Diff line number Diff line change
Expand Up @@ -434,6 +434,8 @@ def get_test_query_count(model, implementation, device, num_devices=1):
return p_range

def get_min_system_requirements(spaces, model, implementation, device):
if implementation.lower() not in ['reference', 'nvidia']:
return ""
model = model.lower()
min_sys_req_content = ""
min_sys_req_content += f"{spaces}<details>\n"
Expand All @@ -445,42 +447,50 @@ def get_min_system_requirements(spaces, model, implementation, device):
if implementation.lower() == "nvidia":
if "dlrm" in model:
device_memory = "24GB"
elif "llama2-70b" in model or "mixtral" in model:
elif "llama2-70b" in model:
device_memory = "2x80GB"
elif "mixtral" in model:
device_memory = "80GB"
elif "sdxl" in model or "gptj" in model:
device_memory = "16GB"
else:
device_memory = "8GB"
device_memory = "To be updated"
elif implementation.lower() == "reference":
if "dlrm" in model:
device_memory = "2x80GB"
elif "llama2-70b" in model:
device_memory = "8x80GB"
elif "mixtral" in model:
device_memory = "4x80GB"
elif "sdxl" in model:
device_memory = "24GB(fp32), 16GB(fp16)"
elif "gptj" in model:
device_memory = "80GB(fp32). 40GB(fp16)"
elif "pointpainting" in model:
device_memory = "To be updated"
else:
device_memory = "8GB"
device_memory = "To be updated"
min_sys_req_content += f"{spaces}* **Device Memory**: {device_memory}\n\n"
# disk space
if "dlrm" in model:
disk_space = "500GB"
elif "llama2-70b" in model:
disk_space = "700GB"
elif "mixtral" in model:
disk_space = "100GB"
elif "retinanet" in model:
disk_space = "200GB"
elif "pointpainting" in model:

if implementation.lower() == "reference":
# disk space
ds = {
"dlrm": "500GB",
"pointpainting": "500GB",
"llama2-70b": "600GB",
"llama3_1-405b": "2.3TB",
"mixtral": "100GB",
"retinanet": "200GB",
"gptj": "50GB",
"resnet50": "50GB",
"sdxl": "50GB",
"3d-unet": "60GB",
"rgat": "2.3TB",
"bert": "50GB"
}
disk_space = "To be updated"
else:
disk_space = "50GB"
min_sys_req_content += f"{spaces}* **Disk Space**: {disk_space}\n\n"
for key in ds:
if key in model:
disk_space = ds[key]
break

min_sys_req_content += f"{spaces}* **Disk Space**: {disk_space}\n\n"
# System memory
if "dlrm" in model:
system_memory = "512GB"
Expand Down Expand Up @@ -680,6 +690,7 @@ def mlperf_inference_run_command(
f"\\\n{pre_space} --server_target_qps=<SERVER_TARGET_QPS>"
)

rerun = "--rerun" if execution_mode == "test" else ""
run_cmd_extra = get_run_cmd_extra(
f_pre_space,
model,
Expand Down Expand Up @@ -737,7 +748,7 @@ def mlperf_inference_run_command(
{pre_space} --framework={framework} \\
{pre_space} --category={category} {scenario_option} \\
{pre_space} --execution_mode=test \\
{pre_space} --device={device} {docker_cmd_suffix}
{pre_space} --device={device} {docker_cmd_suffix} {rerun}
{f_pre_space}```\n"""

return docker_setup_cmd + run_cmd_extra
Expand Down Expand Up @@ -783,7 +794,7 @@ def mlperf_inference_run_command(
{pre_space} --framework={framework} \\
{pre_space} --category={category} {scenario_option} \\
{pre_space} --execution_mode={execution_mode} \\
{pre_space} --device={device} {cmd_suffix}
{pre_space} --device={device} {cmd_suffix} {rerun}
{f_pre_space}```\n"""

return run_cmd + run_cmd_extra