From 4bcd832130bc494f8d242960224b22a6bb165eee Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 5 Mar 2026 18:58:48 +0530 Subject: [PATCH 01/53] change in model for llama3 automotive --- .../app-mlperf-automotive-mlcommons-python/meta.yaml | 8 ++++---- script/app-mlperf-automotive/meta.yaml | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/script/app-mlperf-automotive-mlcommons-python/meta.yaml b/script/app-mlperf-automotive-mlcommons-python/meta.yaml index 8491fe39b..ea355b660 100644 --- a/script/app-mlperf-automotive-mlcommons-python/meta.yaml +++ b/script/app-mlperf-automotive-mlcommons-python/meta.yaml @@ -566,7 +566,7 @@ variations: names: - ml-model-deeplabv3-plus - llama3_2-3b: + llama3_1-8b: group: models add_deps_recursive: pytorch: @@ -582,12 +582,12 @@ variations: - "yes" names: - dataset-mmlu - - tags: get,ml-model,llama3,_3b,_meta-llama/Llama-3.2-8B-Instruct,_hf + - tags: get,ml-model,llama3,_mlc,_8b,_r2-downloader skip_if_env: MLC_RUN_STATE_DOCKER: - "yes" names: - - ml-model-llama3_2 + - ml-model-llama3_1-8b # Target devices cpu: @@ -644,7 +644,7 @@ variations: MLC_MLPERF_SINGLESTREAM_TARGET_LATENCY_PERCENTILE: "99.9" MLC_MLPERF_DEFAULT_MAX_QUERY_COUNT: 6636 - llama3_2-3b,singlestream: + llama3_1-8b,singlestream: default_env: MLC_MLPERF_SINGLESTREAM_TARGET_LATENCY_PERCENTILE: "90" diff --git a/script/app-mlperf-automotive/meta.yaml b/script/app-mlperf-automotive/meta.yaml index 212120669..9a52537ae 100644 --- a/script/app-mlperf-automotive/meta.yaml +++ b/script/app-mlperf-automotive/meta.yaml @@ -369,13 +369,13 @@ variations: - cognata-deeplab-accuracy-script tags: run,accuracy,mlperf,_cognata_deeplab - llama3_2-3b: + llama3_1-8b: group: models default_env: MLC_USE_DATASET_FROM_HOST: yes env: - MLC_MODEL: llama3_2-3b + MLC_MODEL: llama3_1-8b docker: deps: - tags: get-dataset-mmlu @@ -384,18 +384,18 @@ variations: enable_if_env: MLC_USE_DATASET_FROM_HOST: - "yes" - - tags: get,ml-model,llama3,_3b,_meta-llama/Llama-3.2-8B-Instruct,_hf + - tags: get,ml-model,llama3,_mlc,_8b,_r2-downloader enable_if_env: MLC_USE_MODEL_FROM_HOST: - "yes" names: - - ml-model-llama3_2 + - ml-model-llama3_1-8b mounts: - "${{ MLC_DATASET_MMLU_PATH }}:${{ MLC_DATASET_MMLU_PATH }}" - "${{ LLAMA3_CHECKPOINT_PATH }}:${{ LLAMA3_CHECKPOINT_PATH }}" add_deps_recursive: abtf-inference-implementation: - tags: _llama3_2-3b + tags: _llama3_1-8b posthook_deps: - enable_if_env: MLC_MLPERF_LOADGEN_MODE: From 58e63b26dbab079037d3068fca8fb9841100d850 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 5 Mar 2026 22:24:55 +0530 Subject: [PATCH 02/53] fix typo --- script/app-mlperf-automotive-mlcommons-python/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-automotive-mlcommons-python/customize.py b/script/app-mlperf-automotive-mlcommons-python/customize.py index be18466dd..1f39cbfb4 100644 --- a/script/app-mlperf-automotive-mlcommons-python/customize.py +++ b/script/app-mlperf-automotive-mlcommons-python/customize.py @@ -62,7 +62,7 @@ def preprocess(i): env['MLC_NUM_THREADS'] = env.get('MLC_HOST_CPU_TOTAL_CORES', '1') if env.get('MLC_MLPERF_LOADGEN_MAX_BATCHSIZE', '') != '' and not env.get( - 'MLC_MLPERF_MODEL_SKIP_BATCHING', False) and env['MLC_MODEL'] != "llama3_2-3b": + 'MLC_MLPERF_MODEL_SKIP_BATCHING', False) and env['MLC_MODEL'] != "llama3_1-8b": env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --max-batchsize " + \ str(env['MLC_MLPERF_LOADGEN_MAX_BATCHSIZE']) From 95935e5c8022f7d886c32552785d858e9ebb76cb Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 5 Mar 2026 22:32:13 +0530 Subject: [PATCH 03/53] fix model name --- script/app-mlperf-automotive-mlcommons-python/customize.py | 6 +++--- script/get-mlperf-automotive-src/customize.py | 2 +- script/get-mlperf-automotive-src/meta.yaml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/script/app-mlperf-automotive-mlcommons-python/customize.py b/script/app-mlperf-automotive-mlcommons-python/customize.py index 1f39cbfb4..a586d84a7 100644 --- a/script/app-mlperf-automotive-mlcommons-python/customize.py +++ b/script/app-mlperf-automotive-mlcommons-python/customize.py @@ -71,7 +71,7 @@ def preprocess(i): str(env['MLC_MLPERF_LOADGEN_BATCH_SIZE']) if env.get('MLC_MLPERF_LOADGEN_QUERY_COUNT', '') != '' and not env.get( - 'MLC_TMP_IGNORE_MLPERF_QUERY_COUNT', False) and env.get('MLC_MLPERF_RUN_STYLE', '') != "valid" and env['MLC_MODEL'] != "llama3_2-3b": + 'MLC_TMP_IGNORE_MLPERF_QUERY_COUNT', False) and env.get('MLC_MLPERF_RUN_STYLE', '') != "valid" and env['MLC_MODEL'] != "llama3_1-8b": env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --count " + \ env['MLC_MLPERF_LOADGEN_QUERY_COUNT'] @@ -284,8 +284,8 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options, cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(run_dir, "main.py")} --output {env['OUTPUT_DIR']} --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} --backend {backend} --dataset cognata --device {"cuda" if device == "gpu" else "cpu"} --dataset-path {env['MLC_PREPROCESSED_DATASET_COGNATA_PATH']} --checkpoint {env['MLC_ML_MODEL_DEEPLABV3_PLUS_PATH']} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {scenario_extra_options} {mode_extra_options} {dataset_options}""" - elif env['MLC_MODEL'] in ['llm', 'llama3_2-3b']: - run_dir = env['MLC_MLPERF_INFERENCE_LLAMA3_2_3B_PATH'] + elif env['MLC_MODEL'] in ['llama3_1-8b']: + run_dir = env['MLC_MLPERF_INFERENCE_LLAMA3_1_8B_PATH'] env['RUN_DIR'] = run_dir diff --git a/script/get-mlperf-automotive-src/customize.py b/script/get-mlperf-automotive-src/customize.py index a8ddcc1d5..6c302e9b3 100644 --- a/script/get-mlperf-automotive-src/customize.py +++ b/script/get-mlperf-automotive-src/customize.py @@ -96,7 +96,7 @@ def postprocess(i): automotive_root, 'automotive', '2d-object-detection') env['MLC_MLPERF_INFERENCE_DEEPLABV3PLUS_PATH'] = os.path.join( automotive_root, 'automotive', 'semantic-segmentation') - env['MLC_MLPERF_INFERENCE_LLAMA3_2_3B_PATH'] = os.path.join( + env['MLC_MLPERF_INFERENCE_LLAMA3_1_8B_PATH'] = os.path.join( automotive_root, 'automotive', 'llm') env['MLC_GET_DEPENDENT_CACHED_PATH'] = automotive_root diff --git a/script/get-mlperf-automotive-src/meta.yaml b/script/get-mlperf-automotive-src/meta.yaml index 9f1d3970a..d355b40da 100644 --- a/script/get-mlperf-automotive-src/meta.yaml +++ b/script/get-mlperf-automotive-src/meta.yaml @@ -19,7 +19,7 @@ new_env_keys: - MLC_MLPERF_INFERENCE_BEVFORMER_PATH - MLC_MLPERF_INFERENCE_SSD_RESNET50_PATH - MLC_MLPERF_INFERENCE_DEEPLABV3PLUS_PATH -- MLC_MLPERF_INFERENCE_LLAMA3_2_3B_PATH +- MLC_MLPERF_INFERENCE_LLAMA3_1_8B_PATH - MLC_MLPERF_LAST_RELEASE - MLC_MLPERF_INFERENCE_SOURCE - MLC_MLPERF_INFERENCE_SOURCE_VERSION From b70bc8085d1c8872ca7ffd00cc5d2f6bd5d81ea9 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 6 Mar 2026 11:56:00 +0530 Subject: [PATCH 04/53] Add v1.0 tag --- script/app-mlperf-automotive/meta.yaml | 7 +++++++ script/get-mlperf-automotive-src/customize.py | 4 +++- script/get-mlperf-automotive-src/meta.yaml | 5 ++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/script/app-mlperf-automotive/meta.yaml b/script/app-mlperf-automotive/meta.yaml index 9a52537ae..7948776bb 100644 --- a/script/app-mlperf-automotive/meta.yaml +++ b/script/app-mlperf-automotive/meta.yaml @@ -440,6 +440,8 @@ variations: v0.5: {} + v1.0: {} + mvp-demo: {} poc-demo: {} @@ -449,6 +451,11 @@ variations: base_image: ubuntu:22.04 os_version: 22.04 + v1.0,mlcommons-python,cpu: + docker: + base_image: ubuntu:22.04 + os_version: 22.04 + # Loadgen scenarios offline: env: diff --git a/script/get-mlperf-automotive-src/customize.py b/script/get-mlperf-automotive-src/customize.py index 6c302e9b3..b0fb9ebca 100644 --- a/script/get-mlperf-automotive-src/customize.py +++ b/script/get-mlperf-automotive-src/customize.py @@ -52,7 +52,7 @@ def preprocess(i): env["MLC_GIT_URL"] = "https://github.com/mlcommons/mlperf_automotive" if env.get("MLC_MLPERF_LAST_RELEASE", '') == '': - env["MLC_MLPERF_LAST_RELEASE"] = "v0.5" + env["MLC_MLPERF_LAST_RELEASE"] = "v1.0" if 'MLC_GIT_DEPTH' not in env: env['MLC_GIT_DEPTH'] = '' @@ -98,6 +98,8 @@ def postprocess(i): automotive_root, 'automotive', 'semantic-segmentation') env['MLC_MLPERF_INFERENCE_LLAMA3_1_8B_PATH'] = os.path.join( automotive_root, 'automotive', 'llm') + env['MLC_MLPERF_INFERENCE_UNIAD_PATH'] = os.path.join( + automotive_root, 'automotive', 'end-to-end') env['MLC_GET_DEPENDENT_CACHED_PATH'] = automotive_root diff --git a/script/get-mlperf-automotive-src/meta.yaml b/script/get-mlperf-automotive-src/meta.yaml index d355b40da..49d7e1f44 100644 --- a/script/get-mlperf-automotive-src/meta.yaml +++ b/script/get-mlperf-automotive-src/meta.yaml @@ -96,9 +96,12 @@ variations: MLC_GIT_SUBMODULES: '#' versions: custom: + env: + MLC_MLPERF_LAST_RELEASE: v1.0 + v0.5: env: MLC_MLPERF_LAST_RELEASE: v0.5 master: env: - MLC_MLPERF_LAST_RELEASE: v0.5 + MLC_MLPERF_LAST_RELEASE: v1.0 MLC_TMP_GIT_CHECKOUT: master From 0fb4ce49e6e3d5795cc5e587317ea5ed1172941b Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 6 Mar 2026 11:58:19 +0530 Subject: [PATCH 05/53] Update tag --- script/run-mlperf-automotive-app/meta.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/script/run-mlperf-automotive-app/meta.yaml b/script/run-mlperf-automotive-app/meta.yaml index 619f37b3f..178f80e76 100644 --- a/script/run-mlperf-automotive-app/meta.yaml +++ b/script/run-mlperf-automotive-app/meta.yaml @@ -224,6 +224,11 @@ variations: env: MLC_MLPERF_INFERENCE_VERSION: v0.5 + v1.0: + group: benchmark-version + env: + MLC_MLPERF_INFERENCE_VERSION: v1.0 + performance-and-accuracy: default: true base: From 0f6da5b1cb07392b2e834ec611f7426506d811cf Mon Sep 17 00:00:00 2001 From: Arjun Date: Fri, 6 Mar 2026 15:28:35 +0530 Subject: [PATCH 06/53] Turn off export_json by default for geekbench --- script/README.md | 2 +- .../benchmark-program-geekbench/customize.py | 79 +++++++++---------- script/benchmark-program-geekbench/meta.yaml | 1 + 3 files changed, 41 insertions(+), 41 deletions(-) diff --git a/script/README.md b/script/README.md index 9a387e46e..141fb4569 100644 --- a/script/README.md +++ b/script/README.md @@ -1,6 +1,6 @@ # MLCommons Automation Scripts -*Last updated: 2026-03-06 02:02:38* +*Last updated: 2026-03-06 15:28:35* This directory contains automation scripts for MLPerf benchmarks, AI/ML workflows, and development operations. diff --git a/script/benchmark-program-geekbench/customize.py b/script/benchmark-program-geekbench/customize.py index 109c23862..07b72d867 100644 --- a/script/benchmark-program-geekbench/customize.py +++ b/script/benchmark-program-geekbench/customize.py @@ -44,17 +44,15 @@ def preprocess(i): args.append('--no-upload') # Export results as JSON - results_dir = env.get('MLC_GEEKBENCH_RESULTS_DIR', os.getcwd()) - os.makedirs(results_dir, exist_ok=True) - results_file = os.path.join(results_dir, 'geekbench_results.json') - args.append('--export-json') - if os_info['platform'] == 'windows': + if is_true(env.get('MLC_GEEKBENCH_EXPORT_JSON', 'yes')): + results_dir = env.get('MLC_GEEKBENCH_RESULTS_DIR', os.getcwd()) + os.makedirs(results_dir, exist_ok=True) + results_file = os.path.join(results_dir, 'geekbench_results.json') + args.append('--export-json') args.append(f'"{results_file}"') - else: - args.append(f"'{results_file}'") - env['MLC_GEEKBENCH_RESULTS_FILE'] = results_file - env['MLC_GEEKBENCH_RESULTS_DIR'] = results_dir + env['MLC_GEEKBENCH_RESULTS_FILE'] = results_file + env['MLC_GEEKBENCH_RESULTS_DIR'] = results_dir cmd = f"{q}{geekbench_bin}{q} {' '.join(args)}" @@ -75,36 +73,37 @@ def postprocess(i): results_file = env.get('MLC_GEEKBENCH_RESULTS_FILE', '') - if results_file and os.path.isfile(results_file): - logger.info(f"Geekbench results saved to: {results_file}") - - try: - with open(results_file, 'r') as f: - results = json.load(f) - - # Extract key scores from the results JSON - if 'score' in results: - env['MLC_GEEKBENCH_SCORE'] = str(results['score']) - logger.info(f"Geekbench Score: {results['score']}") - - if 'multicore_score' in results: - env['MLC_GEEKBENCH_MULTICORE_SCORE'] = str( - results['multicore_score']) - logger.info( - f"Geekbench Multi-Core Score: {results['multicore_score']}") - - if 'single_core_score' in results: - env['MLC_GEEKBENCH_SINGLE_CORE_SCORE'] = str( - results['single_core_score']) - logger.info( - f"Geekbench Single-Core Score: {results['single_core_score']}") - - state['geekbench_results'] = results - - except Exception as e: - logger.warning(f"Could not parse Geekbench results: {e}") - else: - logger.warning( - "Geekbench results file not found. The benchmark may not have completed successfully.") + if results_file: + if os.path.isfile(results_file): + logger.info(f"Geekbench results saved to: {results_file}") + + try: + with open(results_file, 'r') as f: + results = json.load(f) + + # Extract key scores from the results JSON + if 'score' in results: + env['MLC_GEEKBENCH_SCORE'] = str(results['score']) + logger.info(f"Geekbench Score: {results['score']}") + + if 'multicore_score' in results: + env['MLC_GEEKBENCH_MULTICORE_SCORE'] = str( + results['multicore_score']) + logger.info( + f"Geekbench Multi-Core Score: {results['multicore_score']}") + + if 'single_core_score' in results: + env['MLC_GEEKBENCH_SINGLE_CORE_SCORE'] = str( + results['single_core_score']) + logger.info( + f"Geekbench Single-Core Score: {results['single_core_score']}") + + state['geekbench_results'] = results + + except Exception as e: + logger.warning(f"Could not parse Geekbench results: {e}") + else: + logger.warning( + "Geekbench results file not found. The benchmark may not have completed successfully.") return {'return': 0} diff --git a/script/benchmark-program-geekbench/meta.yaml b/script/benchmark-program-geekbench/meta.yaml index 4a9af1bdd..6cc12ed78 100644 --- a/script/benchmark-program-geekbench/meta.yaml +++ b/script/benchmark-program-geekbench/meta.yaml @@ -16,6 +16,7 @@ input_mapping: license_key: MLC_GEEKBENCH_LICENSE_KEY workload: MLC_GEEKBENCH_WORKLOAD local_file: MLC_GEEKBENCH_LOCAL_FILE + export_json: MLC_GEEKBENCH_EXPORT_JSON new_env_keys: - MLC_GEEKBENCH_* tags: From 2fb01a0d2f989ae020db133df1a1a90f1afad1db Mon Sep 17 00:00:00 2001 From: Arjun Date: Fri, 6 Mar 2026 15:37:12 +0530 Subject: [PATCH 07/53] Turn off export_json by default for geekbench --- script/benchmark-program-geekbench/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/benchmark-program-geekbench/customize.py b/script/benchmark-program-geekbench/customize.py index 07b72d867..df2016d4e 100644 --- a/script/benchmark-program-geekbench/customize.py +++ b/script/benchmark-program-geekbench/customize.py @@ -44,7 +44,7 @@ def preprocess(i): args.append('--no-upload') # Export results as JSON - if is_true(env.get('MLC_GEEKBENCH_EXPORT_JSON', 'yes')): + if is_true(env.get('MLC_GEEKBENCH_EXPORT_JSON', 'no')): results_dir = env.get('MLC_GEEKBENCH_RESULTS_DIR', os.getcwd()) os.makedirs(results_dir, exist_ok=True) results_file = os.path.join(results_dir, 'geekbench_results.json') From 46f39d38e515aedb62eeb6860f7c7f9f213abe24 Mon Sep 17 00:00:00 2001 From: Arjun Date: Fri, 6 Mar 2026 15:49:45 +0530 Subject: [PATCH 08/53] Turn off export_json by default for geekbench --- script/benchmark-program-geekbench/customize.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/script/benchmark-program-geekbench/customize.py b/script/benchmark-program-geekbench/customize.py index df2016d4e..12b0cd7d2 100644 --- a/script/benchmark-program-geekbench/customize.py +++ b/script/benchmark-program-geekbench/customize.py @@ -43,15 +43,17 @@ def preprocess(i): if is_true(env.get('MLC_GEEKBENCH_NO_UPLOAD', 'yes')): args.append('--no-upload') + results_dir = env.get('MLC_GEEKBENCH_RESULTS_DIR', os.getcwd()) + # Export results as JSON if is_true(env.get('MLC_GEEKBENCH_EXPORT_JSON', 'no')): - results_dir = env.get('MLC_GEEKBENCH_RESULTS_DIR', os.getcwd()) os.makedirs(results_dir, exist_ok=True) results_file = os.path.join(results_dir, 'geekbench_results.json') args.append('--export-json') args.append(f'"{results_file}"') env['MLC_GEEKBENCH_RESULTS_FILE'] = results_file + logger.info(f"Results will be saved to: {results_file}") env['MLC_GEEKBENCH_RESULTS_DIR'] = results_dir cmd = f"{q}{geekbench_bin}{q} {' '.join(args)}" @@ -60,7 +62,6 @@ def preprocess(i): env['MLC_RUN_DIR'] = results_dir logger.info(f"Geekbench command: {cmd}") - logger.info(f"Results will be saved to: {results_file}") return {'return': 0} From 820d29a14d7331f6cabfa70f196dbc77bbf64b92 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 6 Mar 2026 10:20:12 +0000 Subject: [PATCH 09/53] [Automated Commit] Format Codebase [skip ci] --- script/benchmark-program-geekbench/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/benchmark-program-geekbench/customize.py b/script/benchmark-program-geekbench/customize.py index 12b0cd7d2..90e99f926 100644 --- a/script/benchmark-program-geekbench/customize.py +++ b/script/benchmark-program-geekbench/customize.py @@ -44,7 +44,7 @@ def preprocess(i): args.append('--no-upload') results_dir = env.get('MLC_GEEKBENCH_RESULTS_DIR', os.getcwd()) - + # Export results as JSON if is_true(env.get('MLC_GEEKBENCH_EXPORT_JSON', 'no')): os.makedirs(results_dir, exist_ok=True) From 3d6508fa9ecaba958354d71680f64b3a0e58492a Mon Sep 17 00:00:00 2001 From: Arjun Date: Fri, 6 Mar 2026 15:50:51 +0530 Subject: [PATCH 10/53] Turn off export_json by default for geekbench --- script/README.md | 2 +- script/benchmark-program-geekbench/meta.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/script/README.md b/script/README.md index 141fb4569..d78d94224 100644 --- a/script/README.md +++ b/script/README.md @@ -1,6 +1,6 @@ # MLCommons Automation Scripts -*Last updated: 2026-03-06 15:28:35* +*Last updated: 2026-03-06 15:50:52* This directory contains automation scripts for MLPerf benchmarks, AI/ML workflows, and development operations. diff --git a/script/benchmark-program-geekbench/meta.yaml b/script/benchmark-program-geekbench/meta.yaml index 6cc12ed78..263e7b50c 100644 --- a/script/benchmark-program-geekbench/meta.yaml +++ b/script/benchmark-program-geekbench/meta.yaml @@ -3,14 +3,17 @@ automation_alias: script automation_uid: 5b4e0237da074764 category: Benchmarking cache: false + default_env: MLC_GEEKBENCH_WORKLOAD: '' + deps: - tags: detect,os - tags: detect,cpu - names: - get-geekbench tags: get,geekbench + input_mapping: license_email: MLC_GEEKBENCH_LICENSE_EMAIL license_key: MLC_GEEKBENCH_LICENSE_KEY From 5f5b749452da522a6d50f15c55f7b5fe576baa43 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 9 Mar 2026 10:15:06 +0530 Subject: [PATCH 11/53] Prevent ownership restoration --- script/get-preprocessed-dataset-nuscenes/run.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/script/get-preprocessed-dataset-nuscenes/run.sh b/script/get-preprocessed-dataset-nuscenes/run.sh index 1954ef736..330cce1aa 100644 --- a/script/get-preprocessed-dataset-nuscenes/run.sh +++ b/script/get-preprocessed-dataset-nuscenes/run.sh @@ -3,11 +3,11 @@ if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "yes" && "$MLC_DOWNLOAD_TOOL" = "rclone" ]]; then cd "${MLC_PREPROCESSED_DATASET_NUSCENES_PATH}" || exit for f in *.tar.gz; do - tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } + tar --no-same-owner -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } done cd "${MLC_PREPROCESSED_DATASET_NUSCENES_ACC_CHECKER_MIN_FILES_PATH}" || exit for f in *.tar.gz; do - tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } + tar --no-same-owner -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } done cd - || exit fi @@ -15,11 +15,11 @@ fi if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "yes" && "$MLC_DOWNLOAD_TOOL" = "r2-downloader" ]]; then cd "${MLC_PREPROCESSED_DATASET_NUSCENES_PATH}/preprocessed" || exit for f in *.tar.gz; do - tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } + tar --no-same-owner -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } done cd "${MLC_PREPROCESSED_DATASET_NUSCENES_PATH}" || exit for f in *.tar.gz; do - tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } + tar --no-same-owner -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } done cd - || exit fi \ No newline at end of file From be217822a56cc0dfad5ee35545e64bb4eda57b5e Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 9 Mar 2026 10:44:13 +0530 Subject: [PATCH 12/53] meta cleanup --- .../app-mlperf-automotive-mlcommons-python/meta.yaml | 1 - script/app-mlperf-automotive/meta.yaml | 1 - script/get-mlperf-automotive-src/meta.yaml | 3 +++ script/run-mlperf-automotive-app/meta.yaml | 12 +++++++++--- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/script/app-mlperf-automotive-mlcommons-python/meta.yaml b/script/app-mlperf-automotive-mlcommons-python/meta.yaml index ea355b660..2821d65aa 100644 --- a/script/app-mlperf-automotive-mlcommons-python/meta.yaml +++ b/script/app-mlperf-automotive-mlcommons-python/meta.yaml @@ -572,7 +572,6 @@ variations: pytorch: version_min: "2.1.0" deps: - - tags: get,generic-python-lib,_package.accelerate - tags: get,generic-python-lib,_package.transformers version_min: 4.40.0 diff --git a/script/app-mlperf-automotive/meta.yaml b/script/app-mlperf-automotive/meta.yaml index 7948776bb..4e9387777 100644 --- a/script/app-mlperf-automotive/meta.yaml +++ b/script/app-mlperf-automotive/meta.yaml @@ -126,7 +126,6 @@ docker: real_run: false user: mlcuser interactive: True - mlc_repos_off: 'mlc pull repo mlcommons@cm4abtf --branch=poc' pre_run_cmds: - mlc pull repo deps: diff --git a/script/get-mlperf-automotive-src/meta.yaml b/script/get-mlperf-automotive-src/meta.yaml index 49d7e1f44..a2bf6a21d 100644 --- a/script/get-mlperf-automotive-src/meta.yaml +++ b/script/get-mlperf-automotive-src/meta.yaml @@ -101,6 +101,9 @@ versions: v0.5: env: MLC_MLPERF_LAST_RELEASE: v0.5 + v1.0: + env: + MLC_MLPERF_LAST_RELEASE: v1.0 master: env: MLC_MLPERF_LAST_RELEASE: v1.0 diff --git a/script/run-mlperf-automotive-app/meta.yaml b/script/run-mlperf-automotive-app/meta.yaml index 178f80e76..63cdaa172 100644 --- a/script/run-mlperf-automotive-app/meta.yaml +++ b/script/run-mlperf-automotive-app/meta.yaml @@ -6,7 +6,7 @@ automation_uid: 5b4e0237da074764 category: MLPerf Automotive -developers: "[Arjun Suresh](https://www.linkedin.com/in/arjunsuresh), [Grigori Fursin](https://cKnowledge.org/gfursin)" +developers: "[Arjun Suresh](https://www.linkedin.com/in/arjunsuresh), [Grigori Fursin](https://cKnowledge.org/gfursin), [Anandhu S](https://www.linkedin.com/in/anandhu-s-2337661b7/)" clean_output_files: @@ -26,7 +26,7 @@ tags_help: "run-abtf,inference" default_env: MLC_MLPERF_IMPLEMENTATION: reference - MLC_MLPERF_MODEL: retinanet + MLC_MLPERF_MODEL: retinanet # we need to modify the default model to something updated and runs within a reasonable time on CPU. MLC_MLPERF_RUN_STYLE: test input_mapping: @@ -223,12 +223,18 @@ variations: group: benchmark-version env: MLC_MLPERF_INFERENCE_VERSION: v0.5 + adr: + automotive-src: + tags: _version.v0.5 v1.0: group: benchmark-version env: MLC_MLPERF_INFERENCE_VERSION: v1.0 - + adr: + automotive-src: + tags: _version.v1.0 + performance-and-accuracy: default: true base: From d7766840cd3f0a1f092f6a3159956a7d0c60a916 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 9 Mar 2026 11:36:04 +0530 Subject: [PATCH 13/53] prevent the model folder to contain .uri as extension --- script/get-ml-model-llama3/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/get-ml-model-llama3/meta.yaml b/script/get-ml-model-llama3/meta.yaml index 8b7e6e2e1..8603bfdca 100644 --- a/script/get-ml-model-llama3/meta.yaml +++ b/script/get-ml-model-llama3/meta.yaml @@ -82,6 +82,8 @@ variations: r2-downloader: default: true group: download-tool + env: + MLC_DOWNLOAD_FILENAME: <<>> add_deps_recursive: dae: tags: _r2-downloader From 50ec1e8382f3d3c060d2cd0b46cb9835ab49f216 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 9 Mar 2026 11:58:31 +0530 Subject: [PATCH 14/53] fix version --- script/run-mlperf-automotive-app/meta.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/run-mlperf-automotive-app/meta.yaml b/script/run-mlperf-automotive-app/meta.yaml index 63cdaa172..09d1d08f3 100644 --- a/script/run-mlperf-automotive-app/meta.yaml +++ b/script/run-mlperf-automotive-app/meta.yaml @@ -225,7 +225,7 @@ variations: MLC_MLPERF_INFERENCE_VERSION: v0.5 adr: automotive-src: - tags: _version.v0.5 + version: v0.5 v1.0: group: benchmark-version @@ -233,8 +233,8 @@ variations: MLC_MLPERF_INFERENCE_VERSION: v1.0 adr: automotive-src: - tags: _version.v1.0 - + version: v1.0 + performance-and-accuracy: default: true base: From 588cbfc83c4e0c5cb0e8878616a61008265410e9 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 9 Mar 2026 12:51:07 +0530 Subject: [PATCH 15/53] Fix env assignment --- script/get-ml-model-llama3/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-ml-model-llama3/customize.py b/script/get-ml-model-llama3/customize.py index c470efc0b..1ff2362ee 100644 --- a/script/get-ml-model-llama3/customize.py +++ b/script/get-ml-model-llama3/customize.py @@ -9,8 +9,8 @@ def preprocess(i): # skip download and register in cache if the llama3 checkpoint path is # already defined by the user - if env.get('MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH', '') != '': - env['LLAMA3_CHECKPOINT_PATH'] = env['MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH'] + if env.get('LLAMA3_CHECKPOINT_PATH', '') != '': + env['MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH'] = env['LLAMA3_CHECKPOINT_PATH'] return {'return': 0} path = env.get('MLC_OUTDIRNAME', '').strip() From 87600ffe741965e0f44ea2cfdb173ba4c79d8813 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 9 Mar 2026 12:56:38 +0530 Subject: [PATCH 16/53] populate mlc ml model path --- script/get-ml-model-llama3/customize.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/get-ml-model-llama3/customize.py b/script/get-ml-model-llama3/customize.py index 1ff2362ee..1fa371e26 100644 --- a/script/get-ml-model-llama3/customize.py +++ b/script/get-ml-model-llama3/customize.py @@ -32,6 +32,8 @@ def postprocess(i): if env.get('MLC_DOWNLOAD_MODE', '') != "dry": if env.get('MLC_ML_MODEL_PATH', '') != '': env['LLAMA3_CHECKPOINT_PATH'] = env['MLC_ML_MODEL_PATH'] + else: + env['MLC_ML_MODEL_PATH'] = env['LLAMA3_CHECKPOINT_PATH'] env['MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH'] = env['LLAMA3_CHECKPOINT_PATH'] env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH'] From 878cbb1b4b38f42c238c87dfe24e2c1bd8de4fca Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 9 Mar 2026 12:58:42 +0530 Subject: [PATCH 17/53] Use existing path if supplied --- script/get-dataset-mmlu/customize.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/script/get-dataset-mmlu/customize.py b/script/get-dataset-mmlu/customize.py index b3d5939f2..2e7c74e56 100644 --- a/script/get-dataset-mmlu/customize.py +++ b/script/get-dataset-mmlu/customize.py @@ -9,6 +9,11 @@ def preprocess(i): env = i['env'] + if env.get('MLC_DATASET_MMLU_PATH', '') != '': + print( + f"Using user defined MMLU dataset path from environment variable 'MLC_DATASET_MMLU_PATH': {env['MLC_DATASET_MMLU_PATH']}") + return {'return': 0} + if env.get('MLC_DATASET_DOWNLOAD_SRC', '') != "mlcommons": print("Using MLCommons Inference source from '" + env['MLC_MLPERF_INFERENCE_SOURCE'] + "'") From b60f109ecdf0f8b3cdeaeffd22f5ca36f2581f2e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 9 Mar 2026 07:29:06 +0000 Subject: [PATCH 18/53] [Automated Commit] Format Codebase [skip ci] --- script/get-dataset-mmlu/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-dataset-mmlu/customize.py b/script/get-dataset-mmlu/customize.py index 2e7c74e56..288a7ccde 100644 --- a/script/get-dataset-mmlu/customize.py +++ b/script/get-dataset-mmlu/customize.py @@ -13,7 +13,7 @@ def preprocess(i): print( f"Using user defined MMLU dataset path from environment variable 'MLC_DATASET_MMLU_PATH': {env['MLC_DATASET_MMLU_PATH']}") return {'return': 0} - + if env.get('MLC_DATASET_DOWNLOAD_SRC', '') != "mlcommons": print("Using MLCommons Inference source from '" + env['MLC_MLPERF_INFERENCE_SOURCE'] + "'") From ad738d533e90259573d7b301d326f86a9d68d297 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 9 Mar 2026 13:21:40 +0530 Subject: [PATCH 19/53] add sentencepiece and protobuf as dependencies --- .../meta.yaml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/script/app-mlperf-automotive-mlcommons-python/meta.yaml b/script/app-mlperf-automotive-mlcommons-python/meta.yaml index 2821d65aa..6a5dc5286 100644 --- a/script/app-mlperf-automotive-mlcommons-python/meta.yaml +++ b/script/app-mlperf-automotive-mlcommons-python/meta.yaml @@ -575,6 +575,8 @@ variations: - tags: get,generic-python-lib,_package.accelerate - tags: get,generic-python-lib,_package.transformers version_min: 4.40.0 + - tags: get,generic-python-lib,_package.protobuf + - tags: get,generic-python-lib,_package.sentencepiece - tags: get-dataset-mmlu skip_if_env: MLC_RUN_STATE_DOCKER: @@ -588,6 +590,28 @@ variations: names: - ml-model-llama3_1-8b + uniad: + group: models + add_deps_recursive: + pytorch: + version_max: "1.21.1" + version_max_usable: "1.21.1" + deps: + - tags: get,generic-python-lib,_package.opencv_python + version_min: 4.40.0 + - tags: get,preprocessed,dataset,nuscenes,_mlc,_validation + skip_if_env: + MLC_RUN_STATE_DOCKER: + - "yes" + names: + - preprocessed-dataset-nuscenes + - tags: get-ml-model-uniad,_mlc,_r2-downloader + skip_if_env: + MLC_RUN_STATE_DOCKER: + - "yes" + names: + - ml-model-uniad + # Target devices cpu: group: device From c6d211cfa3daa0bc5e2bcc678f2a97f6ac49e8ac Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 9 Mar 2026 13:42:38 +0530 Subject: [PATCH 20/53] add tokenizers --- script/app-mlperf-automotive-mlcommons-python/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/app-mlperf-automotive-mlcommons-python/meta.yaml b/script/app-mlperf-automotive-mlcommons-python/meta.yaml index 6a5dc5286..0ee41b081 100644 --- a/script/app-mlperf-automotive-mlcommons-python/meta.yaml +++ b/script/app-mlperf-automotive-mlcommons-python/meta.yaml @@ -577,6 +577,7 @@ variations: version_min: 4.40.0 - tags: get,generic-python-lib,_package.protobuf - tags: get,generic-python-lib,_package.sentencepiece + - tags: get,generic-python-lib,_package.tokenizers - tags: get-dataset-mmlu skip_if_env: MLC_RUN_STATE_DOCKER: From aa4c878cffc1c1e06bc483af58b2f5c2940b3949 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 9 Mar 2026 14:29:29 +0530 Subject: [PATCH 21/53] Support docker devices --- script/run-docker-container/customize.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 5df63f201..f086fbb9c 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -178,7 +178,10 @@ def postprocess(i): if env.get("MLC_DOCKER_GPU_DEVICES"): for d in env["MLC_DOCKER_GPU_DEVICES"].split(","): - run_opts += f" --gpus device={d}" + if env.get('MLC_CONTAINER_TOOL') == "podman": + run_opts += f" --device nvidia.com/gpu={d}" + else: + run_opts += f" --gpus device={d}" elif env.get('MLC_DOCKER_ADD_NUM_GPUS', '') != '': run_opts += " --gpus={}".format(env['MLC_DOCKER_ADD_NUM_GPUS']) elif env.get('MLC_DOCKER_ADD_ALL_GPUS', '') != '': From 01b3d44883a12d6404c5b78d4d59d9c784d1770e Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 9 Mar 2026 15:50:19 +0530 Subject: [PATCH 22/53] fix for GPU visibility --- script/run-docker-container/customize.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index f086fbb9c..249f73a84 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -177,13 +177,16 @@ def postprocess(i): run_opts += " --privileged " if env.get("MLC_DOCKER_GPU_DEVICES"): - for d in env["MLC_DOCKER_GPU_DEVICES"].split(","): - if env.get('MLC_CONTAINER_TOOL') == "podman": - run_opts += f" --device nvidia.com/gpu={d}" - else: + if env.get('MLC_CONTAINER_TOOL') == "podman": + run_opts += f" -e NVIDIA_VISIBLE_DEVICES={env['MLC_DOCKER_GPU_DEVICES']}" + else: + for d in env["MLC_DOCKER_GPU_DEVICES"].split(","): run_opts += f" --gpus device={d}" elif env.get('MLC_DOCKER_ADD_NUM_GPUS', '') != '': - run_opts += " --gpus={}".format(env['MLC_DOCKER_ADD_NUM_GPUS']) + if env.get('MLC_CONTAINER_TOOL') == "podman": + run_opts += f" --device nvidia.com/gpu={env['MLC_DOCKER_ADD_NUM_GPUS']}" + else: + run_opts += " --gpus={}".format(env['MLC_DOCKER_ADD_NUM_GPUS']) elif env.get('MLC_DOCKER_ADD_ALL_GPUS', '') != '': if env.get('MLC_CONTAINER_TOOL') == "podman": run_opts += " --device nvidia.com/gpu=all" From e93b16280c4014708c323ea41cea1e1f870e00b0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 9 Mar 2026 10:20:45 +0000 Subject: [PATCH 23/53] [Automated Commit] Format Codebase [skip ci] --- script/run-docker-container/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 249f73a84..428be6cd7 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -178,7 +178,7 @@ def postprocess(i): if env.get("MLC_DOCKER_GPU_DEVICES"): if env.get('MLC_CONTAINER_TOOL') == "podman": - run_opts += f" -e NVIDIA_VISIBLE_DEVICES={env['MLC_DOCKER_GPU_DEVICES']}" + run_opts += f" -e NVIDIA_VISIBLE_DEVICES={env['MLC_DOCKER_GPU_DEVICES']}" else: for d in env["MLC_DOCKER_GPU_DEVICES"].split(","): run_opts += f" --gpus device={d}" From 286718d9bb331793d606b5635f22feac1ca19ad7 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 9 Mar 2026 15:58:00 +0530 Subject: [PATCH 24/53] Fix for GPU visibility inside docker --- script/run-docker-container/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 249f73a84..5686f8021 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -178,7 +178,7 @@ def postprocess(i): if env.get("MLC_DOCKER_GPU_DEVICES"): if env.get('MLC_CONTAINER_TOOL') == "podman": - run_opts += f" -e NVIDIA_VISIBLE_DEVICES={env['MLC_DOCKER_GPU_DEVICES']}" + run_opts += f" --device nvidia.com/gpu={len(env['MLC_DOCKER_GPU_DEVICES'].split(','))} -e NVIDIA_VISIBLE_DEVICES={env['MLC_DOCKER_GPU_DEVICES']}" else: for d in env["MLC_DOCKER_GPU_DEVICES"].split(","): run_opts += f" --gpus device={d}" From e3d07070338ee05eacb7491ec2716075eeabe097 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 9 Mar 2026 10:29:57 +0000 Subject: [PATCH 25/53] [Automated Commit] Format Codebase [skip ci] --- script/run-docker-container/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 5686f8021..9d87ac5cb 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -178,7 +178,7 @@ def postprocess(i): if env.get("MLC_DOCKER_GPU_DEVICES"): if env.get('MLC_CONTAINER_TOOL') == "podman": - run_opts += f" --device nvidia.com/gpu={len(env['MLC_DOCKER_GPU_DEVICES'].split(','))} -e NVIDIA_VISIBLE_DEVICES={env['MLC_DOCKER_GPU_DEVICES']}" + run_opts += f" --device nvidia.com/gpu={len(env['MLC_DOCKER_GPU_DEVICES'].split(','))} -e NVIDIA_VISIBLE_DEVICES={env['MLC_DOCKER_GPU_DEVICES']}" else: for d in env["MLC_DOCKER_GPU_DEVICES"].split(","): run_opts += f" --gpus device={d}" From d1f1b5f41996d374c6671c6243699b5730bc35ce Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 9 Mar 2026 16:15:23 +0530 Subject: [PATCH 26/53] Fixes for GPU visibility --- script/run-docker-container/customize.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 5686f8021..732d60927 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -178,7 +178,9 @@ def postprocess(i): if env.get("MLC_DOCKER_GPU_DEVICES"): if env.get('MLC_CONTAINER_TOOL') == "podman": - run_opts += f" --device nvidia.com/gpu={len(env['MLC_DOCKER_GPU_DEVICES'].split(','))} -e NVIDIA_VISIBLE_DEVICES={env['MLC_DOCKER_GPU_DEVICES']}" + run_opts += f" -e NVIDIA_VISIBLE_DEVICES={env['MLC_DOCKER_GPU_DEVICES']}" + for d in env["MLC_DOCKER_GPU_DEVICES"].split(","): + run_opts += f" --device nvidia.com/gpu={d}" else: for d in env["MLC_DOCKER_GPU_DEVICES"].split(","): run_opts += f" --gpus device={d}" From 5c4bac2012d5a769e3036061c4130fde4f844d56 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 9 Mar 2026 10:46:15 +0000 Subject: [PATCH 27/53] [Automated Commit] Format Codebase [skip ci] --- script/run-docker-container/customize.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 732d60927..f0f13da04 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -178,9 +178,9 @@ def postprocess(i): if env.get("MLC_DOCKER_GPU_DEVICES"): if env.get('MLC_CONTAINER_TOOL') == "podman": - run_opts += f" -e NVIDIA_VISIBLE_DEVICES={env['MLC_DOCKER_GPU_DEVICES']}" - for d in env["MLC_DOCKER_GPU_DEVICES"].split(","): - run_opts += f" --device nvidia.com/gpu={d}" + run_opts += f" -e NVIDIA_VISIBLE_DEVICES={env['MLC_DOCKER_GPU_DEVICES']}" + for d in env["MLC_DOCKER_GPU_DEVICES"].split(","): + run_opts += f" --device nvidia.com/gpu={d}" else: for d in env["MLC_DOCKER_GPU_DEVICES"].split(","): run_opts += f" --gpus device={d}" From 8a50583fbce847f1a495469051c63c5caff07ac7 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 9 Mar 2026 17:20:21 +0530 Subject: [PATCH 28/53] Print mmlu path at end --- script/get-dataset-mmlu/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/get-dataset-mmlu/meta.yaml b/script/get-dataset-mmlu/meta.yaml index f9de2bc3f..988fe9dbc 100644 --- a/script/get-dataset-mmlu/meta.yaml +++ b/script/get-dataset-mmlu/meta.yaml @@ -26,6 +26,8 @@ new_env_keys: tags: - get-dataset-mmlu uid: 137c805565214b95 +print_env_at_the_end: + MLC_DATASET_MMLU_PATH: MMLU Dataset path variations: # calibration: # env: From c4cf625b2e5e7b504e59d6d81e020c9d29b4a4a9 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 10 Mar 2026 10:48:02 +0530 Subject: [PATCH 29/53] support outdirname --- script/get-dataset-mmlu/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-dataset-mmlu/customize.py b/script/get-dataset-mmlu/customize.py index 288a7ccde..6bd8dbf97 100644 --- a/script/get-dataset-mmlu/customize.py +++ b/script/get-dataset-mmlu/customize.py @@ -20,10 +20,10 @@ def preprocess(i): downloader_path = os.path.join( env['MLC_MLPERF_INFERENCE_SOURCE'], 'automotive', 'llm', 'download_data.py') env['MLC_DATASET_MMLU_PATH'] = os.path.join( - env.get('MLC_DATASET_MMLU_OUT_PATH', os.getcwd()), "mmlu.json") + env.get('MLC_DATASET_MMLU_OUT_PATH', env.get('MLC_OUTDIRNAME',os.getcwd())), "mmlu.json") run_cmd = f"{env['MLC_PYTHON_BIN_WITH_PATH']} {downloader_path} --output {env['MLC_DATASET_MMLU_PATH']}" if env.get('MLC_DATASET_MMLU_COUNT', '') != '': - run_cmd += f" --count {int(env['MLC_DATASET_MMLU_COUNT'])})" + run_cmd += f" --count {int(env['MLC_DATASET_MMLU_COUNT'])}" env['MLC_RUN_CMD'] = run_cmd From 5a33d92a44a58f70f92c2b04d1a89eef602f7793 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 10 Mar 2026 05:18:34 +0000 Subject: [PATCH 30/53] [Automated Commit] Format Codebase [skip ci] --- script/get-dataset-mmlu/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-dataset-mmlu/customize.py b/script/get-dataset-mmlu/customize.py index 6bd8dbf97..4daeec64f 100644 --- a/script/get-dataset-mmlu/customize.py +++ b/script/get-dataset-mmlu/customize.py @@ -20,7 +20,7 @@ def preprocess(i): downloader_path = os.path.join( env['MLC_MLPERF_INFERENCE_SOURCE'], 'automotive', 'llm', 'download_data.py') env['MLC_DATASET_MMLU_PATH'] = os.path.join( - env.get('MLC_DATASET_MMLU_OUT_PATH', env.get('MLC_OUTDIRNAME',os.getcwd())), "mmlu.json") + env.get('MLC_DATASET_MMLU_OUT_PATH', env.get('MLC_OUTDIRNAME', os.getcwd())), "mmlu.json") run_cmd = f"{env['MLC_PYTHON_BIN_WITH_PATH']} {downloader_path} --output {env['MLC_DATASET_MMLU_PATH']}" if env.get('MLC_DATASET_MMLU_COUNT', '') != '': run_cmd += f" --count {int(env['MLC_DATASET_MMLU_COUNT'])}" From 4cd6ab827b0b4e5862b06d7d28fdfb6d02423874 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 10 Mar 2026 14:01:05 +0530 Subject: [PATCH 31/53] potential fix for submission variation --- script/run-mlperf-automotive-app/meta.yaml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/script/run-mlperf-automotive-app/meta.yaml b/script/run-mlperf-automotive-app/meta.yaml index 09d1d08f3..262d58ef5 100644 --- a/script/run-mlperf-automotive-app/meta.yaml +++ b/script/run-mlperf-automotive-app/meta.yaml @@ -259,10 +259,7 @@ variations: post_deps: - names: - submission-generator - enable_if_env: + skip_if_env: MLC_MLPERF_SKIP_SUBMISSION_GENERATION: - - 'no' - - 'false' - - 'False' - - '0' - tags: generate,mlperf,inference,submission,_automotive + - 'yes' + tags: generate,mlperf,inference,submission,_wg-automotive From 4376a943c6fbb0357f4606d842b98a176e6c3f55 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 10 Mar 2026 14:10:11 +0530 Subject: [PATCH 32/53] fix for submission generation --- script/run-mlperf-automotive-app/meta.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/script/run-mlperf-automotive-app/meta.yaml b/script/run-mlperf-automotive-app/meta.yaml index 262d58ef5..9f83fd8aa 100644 --- a/script/run-mlperf-automotive-app/meta.yaml +++ b/script/run-mlperf-automotive-app/meta.yaml @@ -174,7 +174,6 @@ variations: tags: _full env: MLC_MLPERF_SUBMISSION_GENERATION_STYLE: full - MLC_MLPERF_SKIP_SUBMISSION_GENERATION: 'yes' group: submission-generation-style performance-only: From e169c0fff30a812d6cb4b6009d541f133ca929a7 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 10 Mar 2026 14:46:17 +0530 Subject: [PATCH 33/53] add status arg --- script/run-mlperf-automotive-app/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/run-mlperf-automotive-app/meta.yaml b/script/run-mlperf-automotive-app/meta.yaml index 9f83fd8aa..821445e89 100644 --- a/script/run-mlperf-automotive-app/meta.yaml +++ b/script/run-mlperf-automotive-app/meta.yaml @@ -73,6 +73,7 @@ input_mapping: run_style: MLC_MLPERF_RUN_STYLE scenario: MLC_MLPERF_LOADGEN_SCENARIO server_target_qps: MLC_MLPERF_LOADGEN_SERVER_TARGET_QPS + status: MLC_MLPERF_SUBMISSION_SYSTEM_STATUS constantstream_target_qps: MLC_MLPERF_LOADGEN_CONSTANTSTREAM_TARGET_QPS singlestream_target_latency: MLC_MLPERF_LOADGEN_SINGLESTREAM_TARGET_LATENCY skip_submission_generation: MLC_MLPERF_SKIP_SUBMISSION_GENERATION From 09616efd28f27202b69e7762a5a6ccb6d257ecb3 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 10 Mar 2026 14:52:51 +0530 Subject: [PATCH 34/53] add category as input arg --- script/run-mlperf-automotive-app/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/run-mlperf-automotive-app/meta.yaml b/script/run-mlperf-automotive-app/meta.yaml index 821445e89..4d807e3c7 100644 --- a/script/run-mlperf-automotive-app/meta.yaml +++ b/script/run-mlperf-automotive-app/meta.yaml @@ -91,6 +91,7 @@ input_mapping: threads: MLC_NUM_THREADS batch_size: MLC_MLPERF_LOADGEN_MAX_BATCHSIZE sut: MLC_MLPERF_INFERENCE_SUT_VARIATION + category: MLC_MLPERF_SUBMISSION_SYSTEM_TYPE new_state_keys: - app_mlperf_inference_* From 982fa41eb67e19806beab56f250a24c811e48083 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 10 Mar 2026 14:57:12 +0530 Subject: [PATCH 35/53] add submission checker version --- script/run-mlperf-automotive-app/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/run-mlperf-automotive-app/meta.yaml b/script/run-mlperf-automotive-app/meta.yaml index 4d807e3c7..d35b7655f 100644 --- a/script/run-mlperf-automotive-app/meta.yaml +++ b/script/run-mlperf-automotive-app/meta.yaml @@ -224,6 +224,7 @@ variations: group: benchmark-version env: MLC_MLPERF_INFERENCE_VERSION: v0.5 + MLC_MLPERF_SUBMISSION_CHECKER_VERSION: v0.5 adr: automotive-src: version: v0.5 @@ -232,6 +233,7 @@ variations: group: benchmark-version env: MLC_MLPERF_INFERENCE_VERSION: v1.0 + MLC_MLPERF_SUBMISSION_CHECKER_VERSION: v1.0 adr: automotive-src: version: v1.0 From 350e3d46510b17d6aa3e9f8daa6695ea78d18c55 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 10 Mar 2026 18:52:34 +0530 Subject: [PATCH 36/53] use root as user in podman --- script/build-dockerfile/customize.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 706b4c85d..bf8eb4bac 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -22,6 +22,9 @@ def preprocess(i): with open(os.path.join(path, "dockerinfo.json")) as f: config = json.load(f) + if env['MLC_CONTAINER_TOOL'] == "podman": + config['USER'] = "root" + build_args = [] build_args_default = {} input_args = [] From 19f569ca828ad7ddaaa4cb595bb3ea8f744155b8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 11 Mar 2026 09:24:13 +0000 Subject: [PATCH 37/53] [Automated Commit] Document script/get-dataset-mmlu/meta.yaml [skip ci] --- script/get-dataset-mmlu/README.md | 69 +++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 script/get-dataset-mmlu/README.md diff --git a/script/get-dataset-mmlu/README.md b/script/get-dataset-mmlu/README.md new file mode 100644 index 000000000..90da53397 --- /dev/null +++ b/script/get-dataset-mmlu/README.md @@ -0,0 +1,69 @@ +# README for get-dataset-mmlu +This README is automatically generated. Create and add custom content in info.md. Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get-dataset-mmlu +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--out_path` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--search_folder_path` | The folder path where executables of a given script need to be searched. Search is done recursively upto 4 levels. | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--skip_sudo` | Skip SUDO detection | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Ungrouped + +- `mlc` +- `no-of-samples.#` _(# can be substituted dynamically)_ From 5e4f9f55075c7013f2e883ecf72f23443a932328 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 18 Mar 2026 09:50:44 +0000 Subject: [PATCH 38/53] Convert relative path to absolute path for path inputs --- automation/script/experiment.py | 6 ++++++ automation/script/module.py | 22 ++++++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/automation/script/experiment.py b/automation/script/experiment.py index c50fbac26..00ccbfee9 100644 --- a/automation/script/experiment.py +++ b/automation/script/experiment.py @@ -40,6 +40,12 @@ def experiment_run(self_module, i): return prune_result run_input = prune_result['new_input'] + + if 'env' not in run_input: + run_input['env'] = {} + current_path = os.path.abspath(os.getcwd()) + run_input['env']['MLC_USER_RUN_DIR'] = current_path + if run_input.get('exp'): del (run_input['exp']) diff --git a/automation/script/module.py b/automation/script/module.py index 7ad9b89de..3a0a9f971 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -127,6 +127,11 @@ def __init__(self, action_object, automation_file, run_args={}): self.main_script_force_new_cache = run_args.get( 'new', False) # only set for the initial script being called + current_path = os.path.abspath(os.getcwd()) + r = _update_env(self.env, 'MLC_USER_RUN_DIR', current_path) + if r['return'] > 0: + return r + def init_run_state(self, run_state): run_state = run_state or {} @@ -5568,11 +5573,24 @@ def update_env_from_input_mapping( """ Internal: update env from input and input_mapping """ + for key in input_mapping: if key in inp: if key in input_description and str(input_description[key].get( - 'is_path', '')).lower() in ['1', 'yes', 'on', 'true']: - env[input_mapping[key]] = os.path.expanduser(inp[key]) + 'is_path', '')).lower() in ['1', 'yes', 'on', 'true']: + + # 1. Expand the '~' if it exists + path_val = os.path.expanduser(inp[key]) + + # 2. Check if the path is NOT already absolute + if not os.path.isabs(path_val): + # Safely get the base dir (fallback to current working dir just in case) + base_dir = env.get("MLC_USER_RUN_DIR", os.getcwd()) + + # Join the base dir with the relative path and normalize it + path_val = os.path.abspath(os.path.join(base_dir, path_val)) + + env[input_mapping[key]] = path_val else: env[input_mapping[key]] = inp[key] From 93e0363ae0ad1435148fb3932977fbd124114efa Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 18 Mar 2026 09:52:26 +0000 Subject: [PATCH 39/53] [Automated Commit] Format Codebase [skip ci] --- automation/script/experiment.py | 2 +- automation/script/module.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/automation/script/experiment.py b/automation/script/experiment.py index 00ccbfee9..c9c09d9b4 100644 --- a/automation/script/experiment.py +++ b/automation/script/experiment.py @@ -45,7 +45,7 @@ def experiment_run(self_module, i): run_input['env'] = {} current_path = os.path.abspath(os.getcwd()) run_input['env']['MLC_USER_RUN_DIR'] = current_path - + if run_input.get('exp'): del (run_input['exp']) diff --git a/automation/script/module.py b/automation/script/module.py index 3a0a9f971..1483c1e31 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -5577,18 +5577,20 @@ def update_env_from_input_mapping( for key in input_mapping: if key in inp: if key in input_description and str(input_description[key].get( - 'is_path', '')).lower() in ['1', 'yes', 'on', 'true']: + 'is_path', '')).lower() in ['1', 'yes', 'on', 'true']: # 1. Expand the '~' if it exists path_val = os.path.expanduser(inp[key]) # 2. Check if the path is NOT already absolute if not os.path.isabs(path_val): - # Safely get the base dir (fallback to current working dir just in case) + # Safely get the base dir (fallback to current working dir + # just in case) base_dir = env.get("MLC_USER_RUN_DIR", os.getcwd()) # Join the base dir with the relative path and normalize it - path_val = os.path.abspath(os.path.join(base_dir, path_val)) + path_val = os.path.abspath( + os.path.join(base_dir, path_val)) env[input_mapping[key]] = path_val else: From 7e1aa253e5d8a76bd391056b18cd0b279c9391c9 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 18 Mar 2026 10:45:40 +0000 Subject: [PATCH 40/53] Fix MLC_USER_RUN_DIR usage for experiment run --- automation/script/module.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 1483c1e31..6fdb37fe1 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -127,10 +127,16 @@ def __init__(self, action_object, automation_file, run_args={}): self.main_script_force_new_cache = run_args.get( 'new', False) # only set for the initial script being called - current_path = os.path.abspath(os.getcwd()) - r = _update_env(self.env, 'MLC_USER_RUN_DIR', current_path) - if r['return'] > 0: - return r + if self.env.get('MLC_USER_RUN_DIR', '') == '': + current_path = os.path.abspath(os.getcwd()) + r = _update_env(self.env, 'MLC_USER_RUN_DIR', current_path) + if r['return'] > 0: + return r + + if self.const.get('MLC_USER_RUN_DIR', '') == '': + r = _update_env(self.const, 'MLC_USER_RUN_DIR', self.env['MLC_USER_RUN_DIR']) + if r['return'] > 0: + return r def init_run_state(self, run_state): @@ -5583,10 +5589,8 @@ def update_env_from_input_mapping( path_val = os.path.expanduser(inp[key]) # 2. Check if the path is NOT already absolute - if not os.path.isabs(path_val): - # Safely get the base dir (fallback to current working dir - # just in case) - base_dir = env.get("MLC_USER_RUN_DIR", os.getcwd()) + if not os.path.isabs(path_val) and env.get('MLC_USER_RUN_DIR', '') != '': + base_dir = env["MLC_USER_RUN_DIR"] # Join the base dir with the relative path and normalize it path_val = os.path.abspath( From 7bf5fbddec9b9a08e38af0b1af450d0708f33cc4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 18 Mar 2026 10:47:44 +0000 Subject: [PATCH 41/53] [Automated Commit] Format Codebase [skip ci] --- automation/script/module.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 6fdb37fe1..d6ed1429b 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -132,9 +132,12 @@ def __init__(self, action_object, automation_file, run_args={}): r = _update_env(self.env, 'MLC_USER_RUN_DIR', current_path) if r['return'] > 0: return r - + if self.const.get('MLC_USER_RUN_DIR', '') == '': - r = _update_env(self.const, 'MLC_USER_RUN_DIR', self.env['MLC_USER_RUN_DIR']) + r = _update_env( + self.const, + 'MLC_USER_RUN_DIR', + self.env['MLC_USER_RUN_DIR']) if r['return'] > 0: return r @@ -5589,7 +5592,8 @@ def update_env_from_input_mapping( path_val = os.path.expanduser(inp[key]) # 2. Check if the path is NOT already absolute - if not os.path.isabs(path_val) and env.get('MLC_USER_RUN_DIR', '') != '': + if not os.path.isabs(path_val) and env.get( + 'MLC_USER_RUN_DIR', '') != '': base_dir = env["MLC_USER_RUN_DIR"] # Join the base dir with the relative path and normalize it From 4eb72a233f4df39dbca2ba6c3c230a6ba4394671 Mon Sep 17 00:00:00 2001 From: amd-arsuresh Date: Wed, 18 Mar 2026 15:33:08 +0000 Subject: [PATCH 42/53] Fixed a bug in cache validation (#865) --- .github/pull_request_template.md | 1 - automation/script/cache_utils.py | 15 ++++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 4d5374df0..9d22efdf9 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -16,4 +16,3 @@ - [ ] No secrets or credentials are committed - [ ] Paths, shell commands, and environment handling are safe and portable - diff --git a/automation/script/cache_utils.py b/automation/script/cache_utils.py index a04f0e009..0c4c969a3 100644 --- a/automation/script/cache_utils.py +++ b/automation/script/cache_utils.py @@ -323,8 +323,10 @@ def run_validate_cache_if_present(i, cached_script): ) # reconstruct env/state from cached metadata - env_tmp = copy.deepcopy(i['env']) - state_tmp = copy.deepcopy(i['state']) + env = i['env'] + state = i['state'] + env_saved = copy.deepcopy(env) + state_saved = copy.deepcopy(state) path_to_cached_state_file = os.path.join( cached_script.path, @@ -340,10 +342,10 @@ def run_validate_cache_if_present(i, cached_script): return None new_env = cached_meta.get("new_env", {}) if new_env: - env_tmp.update(new_env) + env.update(new_env) new_state = cached_meta.get("new_state", {}) if new_state: - state_tmp.update(new_state) + state.update(new_state) # re-run deps deps = i['meta'].get('deps') @@ -377,11 +379,14 @@ def run_validate_cache_if_present(i, cached_script): r = i['self'].run_native_script({ 'run_script_input': run_script_input, - 'env': env_tmp, + 'env': env, 'script_name': 'validate_cache', 'detect_version': True }) + i['env'] = env_saved + i['state'] = state_saved + if r['return'] > 0: return None From 07f937a47da7b98eadfbb330d9df43fd8d3168ef Mon Sep 17 00:00:00 2001 From: amd-arsuresh Date: Wed, 18 Mar 2026 20:52:00 +0000 Subject: [PATCH 43/53] Improvements to geekbench run --- automation/script/module.py | 96 +- script/README.md | 2 +- .../benchmark-program-geekbench/customize.py | 1026 ++++++++++++++++- script/benchmark-program-geekbench/meta.yaml | 85 +- script/benchmark-program-geekbench/run.bat | 197 +++- script/benchmark-program-geekbench/run.sh | 164 ++- 6 files changed, 1425 insertions(+), 145 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index d6ed1429b..fe285ce54 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -1002,17 +1002,18 @@ def _run(self, i): if r['return'] > 0: return r - # Check chain of prehook dependencies on other MLC scripts. - # (No execution of customize.py for cached scripts) - logger.debug( - self.recursion_spaces + - ' - Checking prehook dependencies on other MLC scripts:') + if len(prehook_deps): + # Check chain of prehook dependencies on other MLC scripts. + # (No execution of customize.py for cached scripts) + logger.debug( + self.recursion_spaces + + ' - Checking prehook dependencies on other MLC scripts:') - r = self._call_run_deps(prehook_deps, self.local_env_keys, local_env_keys_from_meta, - self.recursion_spaces + extra_recursion_spaces, - variation_tags_string, True, debug_script_tags, show_time, extra_recursion_spaces, run_state) - if r['return'] > 0: - return r + r = self._call_run_deps(prehook_deps, self.local_env_keys, local_env_keys_from_meta, + self.recursion_spaces + extra_recursion_spaces, + variation_tags_string, True, debug_script_tags, show_time, extra_recursion_spaces, run_state) + if r['return'] > 0: + return r # Continue with the selected cached script cached_script = found_cached_scripts[selection] @@ -1052,38 +1053,42 @@ def _run(self, i): 'append_unique': True}) utils.merge_dicts( - {'dict1': new_env, 'dict2': const, 'append_lists': True, 'append_unique': True}) - utils.merge_dicts({'dict1': new_state, + {'dict1': env, 'dict2': const, 'append_lists': True, 'append_unique': True}) + utils.merge_dicts({'dict1': state, 'dict2': const_state, 'append_lists': True, 'append_unique': True}) if not fake_run: - # Check chain of posthook dependencies on other MLC scripts. We consider them same as postdeps when - # script is in cache - logger.debug( - self.recursion_spaces + - ' - Checking posthook dependencies on other MLC scripts:') clean_env_keys_post_deps = meta.get( 'clean_env_keys_post_deps', []) - r = self._call_run_deps(posthook_deps, self.local_env_keys, clean_env_keys_post_deps, - self.recursion_spaces + extra_recursion_spaces, - variation_tags_string, True, debug_script_tags, show_time, extra_recursion_spaces, run_state) - if r['return'] > 0: - return r + # Check chain of posthook dependencies on other MLC scripts. We consider them same as postdeps when + # script is in cache + if len(posthook_deps): + logger.debug( + self.recursion_spaces + + ' - Checking posthook dependencies on other MLC scripts:') + + r = self._call_run_deps(posthook_deps, self.local_env_keys, clean_env_keys_post_deps, + self.recursion_spaces + extra_recursion_spaces, + variation_tags_string, True, debug_script_tags, show_time, extra_recursion_spaces, run_state) + if r['return'] > 0: + return r - logger.debug( - self.recursion_spaces + - ' - Checking post dependencies on other MLC scripts:') + if len(post_deps): + logger.debug( + self.recursion_spaces + + ' - Checking post dependencies on other MLC scripts:') - # Check chain of post dependencies on other MLC scripts - r = self._call_run_deps(post_deps, self.local_env_keys, clean_env_keys_post_deps, - self.recursion_spaces + extra_recursion_spaces, - variation_tags_string, True, debug_script_tags, show_time, extra_recursion_spaces, run_state) - if r['return'] > 0: - return r + # Check chain of post dependencies on other MLC + # scripts + r = self._call_run_deps(post_deps, self.local_env_keys, clean_env_keys_post_deps, + self.recursion_spaces + extra_recursion_spaces, + variation_tags_string, True, debug_script_tags, show_time, extra_recursion_spaces, run_state) + if r['return'] > 0: + return r if renew or (not found_cached and num_found_cached_scripts == 0): # Add more tags to cached tags @@ -1888,19 +1893,6 @@ def _run(self, i): import json logger.info(json.dumps(rr, indent=2)) - if show_time: - logger.info( - self.recursion_spaces + - ' - running time of script "{}": {:.2f} sec.'.format( - ','.join(found_script_tags), - elapsed_time)) - else: - logger.debug( - self.recursion_spaces + - ' - running time of script "{}": {:.2f} sec.'.format( - ','.join(found_script_tags), - elapsed_time)) - if not recursion and show_space: stop_disk_stats = shutil.disk_usage("/") @@ -1922,12 +1914,26 @@ def _run(self, i): v = new_env.get(p, None) - logger.info('{}: {}'.format(t, str(v))) + logger.info(self.recursion_spaces + + ' * {}: {}'.format(t, str(v))) # Check if print nice versions if print_versions: self._print_versions(run_state) + if show_time: + logger.info( + self.recursion_spaces + + ' - running time of script "{}": {:.2f} sec.'.format( + ','.join(found_script_tags), + elapsed_time)) + else: + logger.debug( + self.recursion_spaces + + ' - running time of script "{}": {:.2f} sec.'.format( + ','.join(found_script_tags), + elapsed_time)) + # Check if pause (useful if running a given script in a new terminal # that may close automatically) if i.get('pause', False): diff --git a/script/README.md b/script/README.md index d78d94224..0ae1e3a97 100644 --- a/script/README.md +++ b/script/README.md @@ -1,6 +1,6 @@ # MLCommons Automation Scripts -*Last updated: 2026-03-06 15:50:52* +*Last updated: 2026-03-19 02:02:58* This directory contains automation scripts for MLPerf benchmarks, AI/ML workflows, and development operations. diff --git a/script/benchmark-program-geekbench/customize.py b/script/benchmark-program-geekbench/customize.py index 90e99f926..52cde1ce6 100644 --- a/script/benchmark-program-geekbench/customize.py +++ b/script/benchmark-program-geekbench/customize.py @@ -1,9 +1,140 @@ from mlc import utils import os import json +import csv +import statistics +import datetime from utils import is_true +def compute_olympic_score(values): + """Drop the highest and lowest values, then return the mean of the rest. + If fewer than 3 values, return the plain mean.""" + if not values: + return 0 + if len(values) < 3: + return statistics.mean(values) + sorted_vals = sorted(values) + trimmed = sorted_vals[1:-1] + return statistics.mean(trimmed) + + +def extract_scores(result): + """Extract top-level score fields from a geekbench result dict. + Handles both Geekbench 5 and 6 JSON formats.""" + scores = {} + + # Geekbench 5 / generic format + if 'score' in result: + scores['score'] = result['score'] + if 'multicore_score' in result: + scores['multicore_score'] = result['multicore_score'] + if 'single_core_score' in result: + scores['single_core_score'] = result['single_core_score'] + + # Geekbench 6 nested format + if 'single_core' in result and isinstance(result['single_core'], dict): + sc = result['single_core'].get('score') + if sc is not None: + scores['single_core_score'] = sc + if 'multi_core' in result and isinstance(result['multi_core'], dict): + mc = result['multi_core'].get('score') + if mc is not None: + scores['multicore_score'] = mc + + return scores + + +def extract_workload_scores(result): + """Extract per-section and per-workload scores from a geekbench result dict. + Returns a dict keyed by section name, each containing section score and + a dict of workload name -> {score, runtime, runtimes, runtime_warmup}. + + Geekbench's --iterations N flag results in 1 warmup iteration + (N-1) + scored iterations. The 'runtimes' list contains per-scored-iteration + runtimes, and 'runtime' is their mean. 'runtime_warmup' is the warmup + iteration runtime.""" + sections_data = {} + for section in result.get('sections', []): + sec_name = section.get('name', 'Unknown') + sec_info = { + 'section_score': section.get('score', 0), + 'workloads': {} + } + for wl in section.get('workloads', []): + wl_name = wl.get('name', 'Unknown') + runtimes_list = wl.get('runtimes', []) + sec_info['workloads'][wl_name] = { + 'score': wl.get('score', 0), + 'runtime': wl.get('runtime', 0), + 'runtime_warmup': wl.get('runtime_warmup', 0), + 'runtimes': [round(r, 6) for r in runtimes_list], + 'iterations_scored': len(runtimes_list), + } + sections_data[sec_name] = sec_info + return sections_data + + +def merge_sc_mc_results(sc_result, mc_result): + """Merge single-core and multi-core JSON results into one combined result. + The SC run has only Single-Core section, MC run has only Multi-Core section. + The merged result looks like a normal full run with both sections.""" + merged = dict(sc_result) # Start from SC as base + # Combine sections from both + merged['sections'] = list(sc_result.get( + 'sections', [])) + list(mc_result.get('sections', [])) + # Set top-level scores from both + if 'score' in sc_result: + merged['score'] = sc_result['score'] + if 'multicore_score' in mc_result: + merged['multicore_score'] = mc_result['multicore_score'] + elif 'score' in mc_result: + merged['multicore_score'] = mc_result['score'] + return merged + + +def compute_stats(values): + """Compute mean, median, olympic, stdev, min, max for a list of numeric values.""" + if not values: + return {} + result = { + 'mean': round(statistics.mean(values), 2), + 'median': round(statistics.median(values), 2), + 'olympic': round(compute_olympic_score(values), 2), + 'stdev': round(statistics.stdev(values), 2) if len(values) >= 2 else 0, + 'min': round(min(values), 2), + 'max': round(max(values), 2), + 'count': len(values) + } + if result['mean'] != 0: + result['cv_percent'] = round( + (result['stdev'] / abs(result['mean'])) * 100, 2) + else: + result['cv_percent'] = 0 + return result + + +def _print_table(headers, rows): + """Print a nicely formatted ASCII table.""" + col_widths = [len(str(h)) for h in headers] + for row in rows: + for i, cell in enumerate(row): + col_widths[i] = max(col_widths[i], len(str(cell))) + + fmt = " | ".join(f"{{:<{w}}}" for w in col_widths) + separator = "-+-".join("-" * w for w in col_widths) + + lines = [] + lines.append(fmt.format(*[str(h) for h in headers])) + lines.append(separator) + for row in rows: + lines.append(fmt.format(*[str(c) for c in row])) + + table_str = "\n".join(lines) + print("\n" + table_str) + return table_str + + def preprocess(i): os_info = i['os_info'] @@ -24,44 +155,191 @@ def preprocess(i): if not license_email: return {'return': 1, 'error': 'MLC_GEEKBENCH_LICENSE_EMAIL is required when MLC_GEEKBENCH_LICENSE_KEY is provided'} - # Pass parts separately so shell scripts can construct the command - # without quoting issues env['MLC_GEEKBENCH_LICENSE_KEY'] = license_key env['MLC_GEEKBENCH_LICENSE_EMAIL'] = license_email logger.info( "Geekbench license key provided, will register before benchmark") - # Build the run command + # Results directory + results_dir = env.get('MLC_GEEKBENCH_RESULTS_DIR', os.getcwd()) + os.makedirs(results_dir, exist_ok=True) + env['MLC_GEEKBENCH_RESULTS_DIR'] = results_dir + env['MLC_RUN_DIR'] = results_dir + + # --- Info-only modes (no benchmark execution) --- + if is_true(env.get('MLC_GEEKBENCH_SYSINFO_ONLY', '')): + env['MLC_RUN_CMD'] = f"{q}{geekbench_bin}{q} --sysinfo" + env['MLC_GEEKBENCH_INFO_ONLY_MODE'] = 'yes' + return {'return': 0} + + if is_true(env.get('MLC_GEEKBENCH_GPU_LIST', '')): + env['MLC_RUN_CMD'] = f"{q}{geekbench_bin}{q} --gpu-list" + env['MLC_GEEKBENCH_INFO_ONLY_MODE'] = 'yes' + return {'return': 0} + + if is_true(env.get('MLC_GEEKBENCH_WORKLOAD_LIST', '')): + env['MLC_RUN_CMD'] = f"{q}{geekbench_bin}{q} --workload-list" + env['MLC_GEEKBENCH_INFO_ONLY_MODE'] = 'yes' + return {'return': 0} + + # --- Load mode (display saved result, no benchmark) --- + load_file = env.get('MLC_GEEKBENCH_LOAD_FILE', '').strip() + if load_file: + env['MLC_RUN_CMD'] = f"{q}{geekbench_bin}{q} --load {q}{load_file}{q}" + env['MLC_GEEKBENCH_INFO_ONLY_MODE'] = 'yes' + return {'return': 0} + + # --- Build the base benchmark command --- args = [] - # Workload selection (CPU or compute) + # Workload selection (--cpu, --gpu [API], --compute [API]) workload = env.get('MLC_GEEKBENCH_WORKLOAD', '').strip() if workload: args.append(workload) - # No-upload flag + # Determine single-core / multi-core / all-cores mode + explicit_single_core = is_true(env.get('MLC_GEEKBENCH_SINGLE_CORE', '')) + explicit_multi_core = is_true(env.get('MLC_GEEKBENCH_MULTI_CORE', '')) + all_cores_mode = not explicit_single_core and not explicit_multi_core + + if explicit_single_core: + args.append('--single-core') + elif explicit_multi_core: + args.append('--multi-core') + + # CPU workers (Pro: --cpu-workers N) + cpu_workers = env.get('MLC_GEEKBENCH_CPU_WORKERS', '').strip() + if cpu_workers: + args.append(f'--cpu-workers {cpu_workers}') + + # Section filter (Pro: --section IDs) + section = env.get('MLC_GEEKBENCH_SECTION', '').strip() + if section: + args.append(f'--section {section}') + + # Workload filter (Pro: --workload IDs, used with --section) + workload_ids = env.get('MLC_GEEKBENCH_WORKLOAD_IDS', '').strip() + if workload_ids: + args.append(f'--workload {workload_ids}') + + # Iterations (native geekbench: --iterations N) + iterations = env.get('MLC_GEEKBENCH_ITERATIONS', '3').strip() + args.append(f'--iterations {iterations}') + + # Workload gap (Pro: --workload-gap N milliseconds) + workload_gap = env.get('MLC_GEEKBENCH_WORKLOAD_GAP', '').strip() + if workload_gap: + args.append(f'--workload-gap {workload_gap}') + + # GPU device selection + gpu_platform_id = env.get('MLC_GEEKBENCH_GPU_PLATFORM_ID', '').strip() + if gpu_platform_id: + args.append(f'--gpu-platform-id {gpu_platform_id}') + + gpu_device_id = env.get('MLC_GEEKBENCH_GPU_DEVICE_ID', '').strip() + if gpu_device_id: + args.append(f'--gpu-device-id {gpu_device_id}') + + # Upload control if is_true(env.get('MLC_GEEKBENCH_NO_UPLOAD', 'yes')): args.append('--no-upload') + else: + args.append('--upload') - results_dir = env.get('MLC_GEEKBENCH_RESULTS_DIR', os.getcwd()) + # Extra args passthrough (for any flags not explicitly mapped) + extra_args = env.get('MLC_GEEKBENCH_EXTRA_ARGS', '').strip() + if extra_args: + args.append(extra_args) - # Export results as JSON - if is_true(env.get('MLC_GEEKBENCH_EXPORT_JSON', 'no')): - os.makedirs(results_dir, exist_ok=True) - results_file = os.path.join(results_dir, 'geekbench_results.json') - args.append('--export-json') - args.append(f'"{results_file}"') + # Number of external repeated runs + num_runs = int(env.get('MLC_GEEKBENCH_NUM_RUNS', '1')) + env['MLC_GEEKBENCH_NUM_RUNS'] = str(num_runs) - env['MLC_GEEKBENCH_RESULTS_FILE'] = results_file - logger.info(f"Results will be saved to: {results_file}") - env['MLC_GEEKBENCH_RESULTS_DIR'] = results_dir + # Core pinning + core_pinning = is_true(env.get('MLC_GEEKBENCH_CORE_PINNING', 'no')) + pinned_core = env.get('MLC_GEEKBENCH_PINNED_CORE', '0').strip() + env['MLC_GEEKBENCH_PINNED_CORE'] = pinned_core - cmd = f"{q}{geekbench_bin}{q} {' '.join(args)}" + # Additional export formats (beyond JSON/CSV which are always generated) + extra_exports = "" + save_file = env.get('MLC_GEEKBENCH_SAVE_FILE', '').strip() + if save_file: + extra_exports += f" --save {q}{save_file}{q}" - env['MLC_RUN_CMD'] = cmd - env['MLC_RUN_DIR'] = results_dir + export_html = env.get('MLC_GEEKBENCH_EXPORT_HTML', '').strip() + if export_html: + extra_exports += f" --export-html {q}{export_html}{q}" + + export_xml = env.get('MLC_GEEKBENCH_EXPORT_XML', '').strip() + if export_xml: + extra_exports += f" --export-xml {q}{export_xml}{q}" + + export_text = env.get('MLC_GEEKBENCH_EXPORT_TEXT', '').strip() + if export_text: + extra_exports += f" --export-text {q}{export_text}{q}" - logger.info(f"Geekbench command: {cmd}") + platform = os_info['platform'] + + # --- Split SC/MC mode: core_pinning + all-cores --- + # When core pinning is enabled and neither --single-core nor --multi-core + # was explicitly chosen, we split into two invocations per run: + # 1) Single-core with core pinning + # 2) Multi-core without core pinning + # The shell scripts handle the split and postprocess merges the results. + if core_pinning and all_cores_mode: + env['MLC_GEEKBENCH_SPLIT_SC_MC'] = 'yes' + + # SC command: add --single-core and apply core pinning + sc_args = args + ['--single-core'] + sc_cmd = f"{q}{geekbench_bin}{q} {' '.join(sc_args)}{extra_exports}" + if platform == 'linux': + sc_cmd = f"taskset -c {pinned_core} {sc_cmd}" + + # MC command: add --multi-core, no core pinning + mc_args = args + ['--multi-core'] + mc_cmd = f"{q}{geekbench_bin}{q} {' '.join(mc_args)}{extra_exports}" + + env['MLC_GEEKBENCH_BASE_CMD_SC'] = sc_cmd + env['MLC_GEEKBENCH_BASE_CMD_MC'] = mc_cmd + + if platform == 'windows': + core_num = int(pinned_core) + affinity_mask = f"{1 << core_num:x}" + env['MLC_GEEKBENCH_AFFINITY_MASK'] = affinity_mask + env['MLC_GEEKBENCH_CORE_PINNING'] = 'yes' + + logger.info( + f"Split SC/MC mode enabled with core pinning on core {pinned_core}") + logger.info(f" SC command: {sc_cmd}") + logger.info(f" MC command: {mc_cmd}") + else: + env['MLC_GEEKBENCH_SPLIT_SC_MC'] = 'no' + base_cmd = f"{q}{geekbench_bin}{q} {' '.join(args)}{extra_exports}" + + # Apply core pinning to the single command (only sensible for + # --single-core) + if core_pinning: + if explicit_multi_core: + logger.warning("Core pinning with --multi-core will pin all threads to one core. " + "Consider using split mode (all-cores + core_pinning) instead.") + if platform == 'linux': + base_cmd = f"taskset -c {pinned_core} {base_cmd}" + logger.info( + f"Core pinning enabled (Linux): taskset -c {pinned_core}") + elif platform == 'windows': + core_num = int(pinned_core) + affinity_mask = f"{1 << core_num:x}" + env['MLC_GEEKBENCH_AFFINITY_MASK'] = affinity_mask + env['MLC_GEEKBENCH_CORE_PINNING'] = 'yes' + logger.info( + f"Core pinning enabled (Windows): affinity mask 0x{affinity_mask} (core {core_num})") + + env['MLC_GEEKBENCH_BASE_CMD'] = base_cmd + logger.info(f"Geekbench command: {base_cmd}") + + logger.info( + f"Native iterations per workload: {iterations}, Number of runs: {num_runs}") + logger.info(f"Results directory: {results_dir}") return {'return': 0} @@ -72,39 +350,697 @@ def postprocess(i): logger = i['automation'].logger state = i['state'] - results_file = env.get('MLC_GEEKBENCH_RESULTS_FILE', '') + # Info-only modes — nothing to aggregate + if is_true(env.get('MLC_GEEKBENCH_INFO_ONLY_MODE', '')): + logger.info("Info-only mode — no results to process") + return {'return': 0} - if results_file: - if os.path.isfile(results_file): - logger.info(f"Geekbench results saved to: {results_file}") + results_dir = env.get('MLC_GEEKBENCH_RESULTS_DIR', '') + num_runs = int(env.get('MLC_GEEKBENCH_NUM_RUNS', '1')) + iterations_cfg = env.get('MLC_GEEKBENCH_ITERATIONS', '3') + split_mode = is_true(env.get('MLC_GEEKBENCH_SPLIT_SC_MC', 'no')) - try: - with open(results_file, 'r') as f: - results = json.load(f) + all_results = [] + run_durations = [] - # Extract key scores from the results JSON - if 'score' in results: - env['MLC_GEEKBENCH_SCORE'] = str(results['score']) - logger.info(f"Geekbench Score: {results['score']}") + for run in range(1, num_runs + 1): + if split_mode: + # In split mode, we have separate SC and MC result files per run + sc_file = os.path.join(results_dir, f'geekbench_run{run}_sc.json') + mc_file = os.path.join(results_dir, f'geekbench_run{run}_mc.json') - if 'multicore_score' in results: - env['MLC_GEEKBENCH_MULTICORE_SCORE'] = str( - results['multicore_score']) - logger.info( - f"Geekbench Multi-Core Score: {results['multicore_score']}") + sc_result = None + mc_result = None - if 'single_core_score' in results: - env['MLC_GEEKBENCH_SINGLE_CORE_SCORE'] = str( - results['single_core_score']) - logger.info( - f"Geekbench Single-Core Score: {results['single_core_score']}") + if os.path.isfile(sc_file): + try: + with open(sc_file, 'r') as f: + sc_result = json.load(f) + logger.info(f"Loaded SC results: run {run}") + except Exception as e: + logger.warning(f"Could not parse {sc_file}: {e}") + else: + logger.warning(f"SC results file not found: {sc_file}") - state['geekbench_results'] = results + if os.path.isfile(mc_file): + try: + with open(mc_file, 'r') as f: + mc_result = json.load(f) + logger.info(f"Loaded MC results: run {run}") + except Exception as e: + logger.warning(f"Could not parse {mc_file}: {e}") + else: + logger.warning(f"MC results file not found: {mc_file}") - except Exception as e: - logger.warning(f"Could not parse Geekbench results: {e}") + # Merge SC + MC into one combined result + if sc_result and mc_result: + result = merge_sc_mc_results(sc_result, mc_result) + result['_run'] = run + result['_split_mode'] = True + all_results.append(result) + # Save merged JSON for reference + merged_file = os.path.join( + results_dir, f'geekbench_run{run}.json') + with open(merged_file, 'w') as f: + json.dump(result, f, indent=2) + logger.info(f"Merged SC+MC results saved: {merged_file}") + elif sc_result: + sc_result['_run'] = run + all_results.append(sc_result) + logger.warning(f"Run {run}: only SC results available") + elif mc_result: + mc_result['_run'] = run + all_results.append(mc_result) + logger.warning(f"Run {run}: only MC results available") + + # Load timing data for both SC and MC + for phase in ['sc', 'mc']: + timing_file = os.path.join( + results_dir, f'geekbench_run{run}_{phase}_timing.json') + if os.path.isfile(timing_file): + try: + with open(timing_file, 'r') as f: + timing = json.load(f) + duration = timing.get('duration_sec', None) + if duration is not None: + run_durations.append({ + 'run': run, + 'phase': phase.upper(), + 'duration_sec': float(duration) + }) + except Exception as e: + logger.warning(f"Could not parse {timing_file}: {e}") + + # Also load total run timing + timing_file = os.path.join( + results_dir, f'geekbench_run{run}_timing.json') + if os.path.isfile(timing_file): + try: + with open(timing_file, 'r') as f: + timing = json.load(f) + duration = timing.get('duration_sec', None) + if duration is not None: + run_durations.append({ + 'run': run, + 'phase': 'total', + 'duration_sec': float(duration) + }) + except Exception as e: + logger.warning(f"Could not parse {timing_file}: {e}") else: - logger.warning( - "Geekbench results file not found. The benchmark may not have completed successfully.") + json_file = os.path.join(results_dir, f'geekbench_run{run}.json') + + if os.path.isfile(json_file): + try: + with open(json_file, 'r') as f: + result = json.load(f) + result['_run'] = run + all_results.append(result) + logger.info(f"Loaded results: run {run}") + except Exception as e: + logger.warning(f"Could not parse {json_file}: {e}") + else: + logger.warning(f"Results file not found: {json_file}") + + # Load timing data + timing_file = os.path.join( + results_dir, f'geekbench_run{run}_timing.json') + if os.path.isfile(timing_file): + try: + with open(timing_file, 'r') as f: + timing = json.load(f) + duration = timing.get('duration_sec', None) + if duration is not None: + run_durations.append({ + 'run': run, + 'phase': 'total', + 'duration_sec': float(duration) + }) + except Exception as e: + logger.warning(f"Could not parse {timing_file}: {e}") + + if not all_results: + logger.warning( + "No Geekbench results found. The benchmark may not have completed successfully.") + return {'return': 0} + + # --- Collect top-level scores across runs --- + overall_score_lists = {} + for r in all_results: + for key, val in extract_scores(r).items(): + overall_score_lists.setdefault(key, []).append(val) + + overall_stats = { + key: compute_stats(vals) for key, vals in overall_score_lists.items() + } + + # --- Collect per-workload scores across runs --- + workload_data = {} + section_scores = {} + for r in all_results: + sections = extract_workload_scores(r) + for sec_name, sec_info in sections.items(): + section_scores.setdefault( + sec_name, []).append( + sec_info['section_score']) + if sec_name not in workload_data: + workload_data[sec_name] = {} + for wl_name, wl_info in sec_info['workloads'].items(): + if wl_name not in workload_data[sec_name]: + workload_data[sec_name][wl_name] = { + 'score': [], 'runtime': [], 'runtimes': []} + workload_data[sec_name][wl_name]['score'].append( + wl_info['score']) + workload_data[sec_name][wl_name]['runtime'].append( + wl_info['runtime']) + workload_data[sec_name][wl_name]['runtimes'].append( + wl_info['runtimes']) + + # Compute per-workload statistics + workload_stats = {} + for sec_name, wls in workload_data.items(): + workload_stats[sec_name] = { + 'section_score_stats': compute_stats(section_scores.get(sec_name, [])), + 'workloads': {} + } + for wl_name, wl_lists in wls.items(): + all_iter_runtimes = [] + for run_runtimes in wl_lists['runtimes']: + all_iter_runtimes.extend(run_runtimes) + workload_stats[sec_name]['workloads'][wl_name] = { + 'score': compute_stats(wl_lists['score']), + 'runtime': compute_stats(wl_lists['runtime']), + 'iteration_runtimes': compute_stats(all_iter_runtimes), + } + + # --- Runtime statistics --- + # For split mode, compute separate SC/MC/total stats + runtime_stats = {} + total_durations = [d['duration_sec'] + for d in run_durations if d['phase'] == 'total'] + if total_durations: + runtime_stats['total'] = compute_stats(total_durations) + if split_mode: + sc_durations = [d['duration_sec'] + for d in run_durations if d['phase'] == 'SC'] + mc_durations = [d['duration_sec'] + for d in run_durations if d['phase'] == 'MC'] + if sc_durations: + runtime_stats['single_core'] = compute_stats(sc_durations) + if mc_durations: + runtime_stats['multi_core'] = compute_stats(mc_durations) + + # --- Build summary --- + individual_results = [] + for r in all_results: + entry = { + 'run': r['_run'], + 'scores': extract_scores(r), + 'sections': extract_workload_scores(r), + } + individual_results.append(entry) + + summary = { + 'num_runs': num_runs, + 'iterations_per_workload': iterations_cfg, + 'total_runs_completed': len(all_results), + 'split_sc_mc': split_mode, + 'overall_statistics': overall_stats, + 'workload_statistics': workload_stats, + 'runtime_statistics': runtime_stats, + 'individual_results': individual_results, + 'individual_runtimes': run_durations, + } + + # Log key results + for key, stat in overall_stats.items(): + logger.info( + f"Overall {key}: mean={stat['mean']}, " + f"median={stat['median']}, olympic={stat['olympic']}, " + f"stdev={stat['stdev']}, cv={stat['cv_percent']}%") + + for phase, rstats in runtime_stats.items(): + logger.info( + f"Runtime {phase} (sec): mean={rstats['mean']}, " + f"stdev={rstats['stdev']}, " + f"cv={rstats['cv_percent']}%, " + f"min={rstats['min']}, max={rstats['max']}") + + # Set env vars with overall mean scores for downstream consumers + if 'score' in overall_stats: + env['MLC_GEEKBENCH_SCORE'] = str(overall_stats['score']['mean']) + if 'single_core_score' in overall_stats: + env['MLC_GEEKBENCH_SINGLE_CORE_SCORE'] = str( + overall_stats['single_core_score']['mean']) + if 'multicore_score' in overall_stats: + env['MLC_GEEKBENCH_MULTICORE_SCORE'] = str( + overall_stats['multicore_score']['mean']) + + # Export summary as JSON + summary_json = os.path.join(results_dir, 'geekbench_summary.json') + with open(summary_json, 'w') as f: + json.dump(summary, f, indent=2) + env['MLC_GEEKBENCH_SUMMARY_JSON'] = summary_json + logger.info(f"Summary JSON saved to: {summary_json}") + + # Export summary as CSV + summary_csv = os.path.join(results_dir, 'geekbench_summary.csv') + _write_summary_csv(summary, summary_csv) + env['MLC_GEEKBENCH_SUMMARY_CSV'] = summary_csv + logger.info(f"Summary CSV saved to: {summary_csv}") + + # --- Print results in tabular format --- + _print_results_table(summary) + + # --- Create benchmark database entry JSON --- + db_json_path = os.path.join(results_dir, 'geekbench_benchmark_entry.json') + _write_benchmark_db_entry(env, summary, db_json_path, logger) + env['MLC_GEEKBENCH_BENCHMARK_ENTRY_JSON'] = db_json_path + + state['geekbench_results'] = all_results + state['geekbench_summary'] = summary return {'return': 0} + + +def _print_results_table(summary): + """Print results in nicely formatted ASCII tables.""" + + individual = summary.get('individual_results', []) + run_durations = summary.get('individual_runtimes', []) + num_runs = summary.get('num_runs', 1) + iterations_cfg = summary.get('iterations_per_workload', '3') + split_mode = summary.get('split_sc_mc', False) + + # Build per-run duration maps + total_dur = {} + sc_dur = {} + mc_dur = {} + for d in run_durations: + run = d['run'] + if d['phase'] == 'total': + total_dur[run] = d['duration_sec'] + elif d['phase'] == 'SC': + sc_dur[run] = d['duration_sec'] + elif d['phase'] == 'MC': + mc_dur[run] = d['duration_sec'] + + # ============================================================ + # 1. Individual Run Top-Level Scores + # ============================================================ + if individual: + score_keys = sorted(individual[0]['scores'].keys()) + headers = ['Run'] + [k.replace('_', ' ').title() for k in score_keys] + if split_mode: + headers.extend(['SC Duration (s)', 'MC Duration (s)', 'Total (s)']) + elif total_dur: + headers.append('Duration (s)') + rows = [] + for entry in individual: + run_num = entry['run'] + row = [run_num] + row.extend(entry['scores'].get(k, '-') for k in score_keys) + if split_mode: + row.append(sc_dur.get(run_num, '-')) + row.append(mc_dur.get(run_num, '-')) + row.append(total_dur.get(run_num, '-')) + elif total_dur: + row.append(total_dur.get(run_num, '-')) + rows.append(row) + + print("") + print("=" * 78) + if split_mode: + print(" INDIVIDUAL RUN SCORES (SC pinned, MC unpinned)") + else: + print(" INDIVIDUAL RUN SCORES (top-level)") + print("=" * 78) + _print_table(headers, rows) + + # ============================================================ + # 2. Per-Section / Per-Workload Results for each run + # ============================================================ + if individual and individual[0].get('sections'): + for entry in individual: + run_num = entry['run'] + sections = entry.get('sections', {}) + for sec_name, sec_info in sections.items(): + print("") + print("-" * 78) + pinned_note = "" + if split_mode: + if 'single' in sec_name.lower(): + pinned_note = " [PINNED]" + else: + pinned_note = " [UNPINNED]" + label = (f" Run {run_num} | {sec_name}{pinned_note}" + f" (section score: {sec_info['section_score']}," + f" iterations: {iterations_cfg})") + print(label) + print("-" * 78) + + max_iters = 0 + for wl_info in sec_info['workloads'].values(): + max_iters = max( + max_iters, wl_info.get( + 'iterations_scored', 0)) + + headers = ['Workload', 'Score', 'Warmup (s)', 'Mean RT (s)'] + for it in range(1, max_iters + 1): + headers.append(f'Iter {it} (s)') + if max_iters >= 2: + headers.append('Iter Stdev') + + rows = [] + for wl_name, wl_info in sec_info['workloads'].items(): + iters = wl_info.get('runtimes', []) + row = [ + wl_name, + wl_info['score'], + round(wl_info.get('runtime_warmup', 0), 4), + round(wl_info['runtime'], 4), + ] + for it in range(max_iters): + if it < len(iters): + row.append(round(iters[it], 6)) + else: + row.append('-') + if max_iters >= 2: + if len(iters) >= 2: + row.append(round(statistics.stdev(iters), 6)) + else: + row.append('-') + rows.append(row) + _print_table(headers, rows) + + # ============================================================ + # 3. Overall Top-Level Statistics (across runs) + # ============================================================ + overall = summary.get('overall_statistics', {}) + runtime_stats = summary.get('runtime_statistics', {}) + + if overall: + headers = [ + 'Metric', + 'Mean', + 'Median', + 'Olympic', + 'Stdev', + 'CV%', + 'Min', + 'Max', + 'Count'] + rows = [] + for metric, stats in overall.items(): + rows.append([ + metric.replace('_', ' ').title(), + stats.get('mean', '-'), + stats.get('median', '-'), + stats.get('olympic', '-'), + stats.get('stdev', '-'), + stats.get('cv_percent', '-'), + stats.get('min', '-'), + stats.get('max', '-'), + stats.get('count', '-'), + ]) + for phase, rstats in runtime_stats.items(): + rows.append([ + f'Runtime {phase} (sec)', + rstats.get('mean', '-'), + rstats.get('median', '-'), + rstats.get('olympic', '-'), + rstats.get('stdev', '-'), + rstats.get('cv_percent', '-'), + rstats.get('min', '-'), + rstats.get('max', '-'), + rstats.get('count', '-'), + ]) + + print("") + print("=" * 78) + print(" OVERALL TOP-LEVEL STATISTICS (across all runs)") + print("=" * 78) + _print_table(headers, rows) + + # ============================================================ + # 4. Per-Workload Statistics (aggregated across runs) + # ============================================================ + workload_stats = summary.get('workload_statistics', {}) + + if workload_stats and num_runs > 1: + for sec_name, sec_info in workload_stats.items(): + sec_stats = sec_info.get('section_score_stats', {}) + pinned_note = "" + if split_mode: + if 'single' in sec_name.lower(): + pinned_note = " [PINNED]" + else: + pinned_note = " [UNPINNED]" + print("") + print("=" * 78) + print(f" WORKLOAD STATISTICS (across runs) | {sec_name}{pinned_note}" + f" (section score: mean={sec_stats.get('mean', '-')}," + f" stdev={sec_stats.get('stdev', '-')}," + f" cv={sec_stats.get('cv_percent', '-')}%)") + print("=" * 78) + headers = ['Workload', + 'Score Mean', 'Score Stdev', 'Score CV%', + 'RT Mean', 'RT Stdev', 'RT CV%', + 'Iter RT Mean', 'Iter RT Stdev', 'Iter RT CV%'] + rows = [] + for wl_name, wl_stats in sec_info['workloads'].items(): + s = wl_stats.get('score', {}) + r = wl_stats.get('runtime', {}) + ir = wl_stats.get('iteration_runtimes', {}) + rows.append([ + wl_name, + s.get('mean', '-'), s.get('stdev', + '-'), s.get('cv_percent', '-'), + r.get('mean', '-'), r.get('stdev', + '-'), r.get('cv_percent', '-'), + ir.get('mean', '-'), ir.get('stdev', + '-'), ir.get('cv_percent', '-'), + ]) + _print_table(headers, rows) + + # For single run, show per-workload with iteration details + if workload_stats and num_runs == 1: + for sec_name, sec_info in workload_stats.items(): + sec_stats = sec_info.get('section_score_stats', {}) + pinned_note = "" + if split_mode: + if 'single' in sec_name.lower(): + pinned_note = " [PINNED]" + else: + pinned_note = " [UNPINNED]" + print("") + print("=" * 78) + print(f" WORKLOAD RESULTS | {sec_name}{pinned_note}" + f" (section score: {sec_stats.get('mean', '-')})") + print("=" * 78) + headers = ['Workload', 'Score', 'Mean RT (s)', + 'Iter RT Stdev', 'Iter RT CV%', 'Iters Scored'] + rows = [] + for wl_name, wl_stats in sec_info['workloads'].items(): + s = wl_stats.get('score', {}) + r = wl_stats.get('runtime', {}) + ir = wl_stats.get('iteration_runtimes', {}) + rows.append([ + wl_name, + s.get('mean', '-'), + r.get('mean', '-'), + ir.get('stdev', '-'), + ir.get('cv_percent', '-'), + ir.get('count', '-'), + ]) + _print_table(headers, rows) + + print("") + + +def _write_summary_csv(summary, csv_path): + """Write summary statistics to a CSV file.""" + split_mode = summary.get('split_sc_mc', False) + + with open(csv_path, 'w', newline='') as f: + writer = csv.writer(f) + + # --- Individual top-level results --- + writer.writerow(['Individual Results (Top-Level Scores)']) + if summary['individual_results']: + score_keys = sorted( + summary['individual_results'][0]['scores'].keys()) + run_durations = summary.get('individual_runtimes', []) + total_dur = {d['run']: d['duration_sec'] + for d in run_durations if d['phase'] == 'total'} + sc_dur = {d['run']: d['duration_sec'] + for d in run_durations if d['phase'] == 'SC'} + mc_dur = {d['run']: d['duration_sec'] + for d in run_durations if d['phase'] == 'MC'} + header = ['run'] + score_keys + if split_mode: + header.extend( + ['sc_duration_sec', 'mc_duration_sec', 'total_duration_sec']) + elif total_dur: + header.append('duration_sec') + writer.writerow(header) + for entry in summary['individual_results']: + run_num = entry['run'] + row = [run_num] + row.extend(entry['scores'].get(k, '') for k in score_keys) + if split_mode: + row.extend([sc_dur.get(run_num, ''), mc_dur.get( + run_num, ''), total_dur.get(run_num, '')]) + elif total_dur: + row.append(total_dur.get(run_num, '')) + writer.writerow(row) + + writer.writerow([]) + + # --- Individual per-workload results --- + for entry in summary.get('individual_results', []): + sections = entry.get('sections', {}) + for sec_name, sec_info in sections.items(): + pinned_note = "" + if split_mode: + pinned_note = " [PINNED]" if 'single' in sec_name.lower( + ) else " [UNPINNED]" + writer.writerow( + [f'Run {entry["run"]} | {sec_name}{pinned_note} (section score: {sec_info["section_score"]})']) + max_iters = max( + (wl['iterations_scored'] + for wl in sec_info['workloads'].values()), + default=0) + header = [ + 'workload', + 'score', + 'warmup_runtime', + 'mean_runtime'] + for it in range(1, max_iters + 1): + header.append(f'iter_{it}_runtime') + if max_iters >= 2: + header.append('iter_stdev') + writer.writerow(header) + for wl_name, wl_info in sec_info['workloads'].items(): + iters = wl_info.get('runtimes', []) + row = [wl_name, wl_info['score'], + round(wl_info.get('runtime_warmup', 0), 6), + round(wl_info['runtime'], 6)] + for it in range(max_iters): + row.append( + round( + iters[it], + 6) if it < len(iters) else '') + if max_iters >= 2: + if len(iters) >= 2: + row.append(round(statistics.stdev(iters), 6)) + else: + row.append('') + writer.writerow(row) + writer.writerow([]) + + # --- Overall statistics --- + writer.writerow(['Overall Statistics']) + writer.writerow(['metric', 'mean', 'median', 'olympic', + 'stdev', 'cv_percent', 'min', 'max', 'count']) + for metric, stats in summary.get('overall_statistics', {}).items(): + writer.writerow([metric, stats.get('mean', ''), stats.get('median', ''), + stats.get('olympic', ''), stats.get('stdev', ''), + stats.get('cv_percent', ''), stats.get('min', ''), + stats.get('max', ''), stats.get('count', '')]) + for phase, rstats in summary.get('runtime_statistics', {}).items(): + writer.writerow([f'runtime_{phase}_sec', rstats.get('mean', ''), + rstats.get( + 'median', ''), rstats.get( + 'olympic', ''), + rstats.get( + 'stdev', ''), rstats.get( + 'cv_percent', ''), + rstats.get('min', ''), rstats.get('max', ''), + rstats.get('count', '')]) + writer.writerow([]) + + # --- Per-workload statistics --- + workload_stats = summary.get('workload_statistics', {}) + if workload_stats: + for sec_name, sec_info in workload_stats.items(): + sec_ss = sec_info.get('section_score_stats', {}) + writer.writerow( + [f'Workload Statistics | {sec_name} (section score mean: {sec_ss.get("mean", "")})']) + writer.writerow(['workload', + 'score_mean', 'score_median', 'score_olympic', + 'score_stdev', 'score_cv_percent', + 'runtime_mean', 'runtime_median', 'runtime_olympic', + 'runtime_stdev', 'runtime_cv_percent', + 'iter_rt_mean', 'iter_rt_stdev', 'iter_rt_cv_percent']) + for wl_name, wl_stats in sec_info['workloads'].items(): + s = wl_stats.get('score', {}) + r = wl_stats.get('runtime', {}) + ir = wl_stats.get('iteration_runtimes', {}) + writer.writerow([ + wl_name, + s.get( + 'mean', ''), s.get( + 'median', ''), s.get( + 'olympic', ''), + s.get('stdev', ''), s.get('cv_percent', ''), + r.get( + 'mean', ''), r.get( + 'median', ''), r.get( + 'olympic', ''), + r.get('stdev', ''), r.get('cv_percent', ''), + ir.get( + 'mean', ''), ir.get( + 'stdev', ''), ir.get( + 'cv_percent', ''), + ]) + writer.writerow([]) + + +def _write_benchmark_db_entry(env, summary, db_json_path, logger): + """Create a JSON file combining benchmark results with platform details.""" + + platform_details = {} + platform_details_file = env.get('MLC_PLATFORM_DETAILS_FILE_PATH', '') + if platform_details_file and os.path.isfile(platform_details_file): + try: + with open(platform_details_file, 'r') as f: + platform_details = json.load(f) + logger.info( + f"Loaded platform details from: {platform_details_file}") + except Exception as e: + logger.warning(f"Could not load platform details: {e}") + else: + logger.warning( + "Platform details file not found. " + "Benchmark DB entry will not include platform information.") + + db_entry = { + 'benchmark': 'geekbench', + 'timestamp': datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'configuration': { + 'workload': env.get('MLC_GEEKBENCH_WORKLOAD', ''), + 'iterations': env.get('MLC_GEEKBENCH_ITERATIONS', '3'), + 'num_runs': env.get('MLC_GEEKBENCH_NUM_RUNS', '1'), + 'core_pinning': env.get('MLC_GEEKBENCH_CORE_PINNING', 'no'), + 'pinned_core': env.get('MLC_GEEKBENCH_PINNED_CORE', '0'), + 'split_sc_mc': env.get('MLC_GEEKBENCH_SPLIT_SC_MC', 'no'), + 'single_core': env.get('MLC_GEEKBENCH_SINGLE_CORE', ''), + 'multi_core': env.get('MLC_GEEKBENCH_MULTI_CORE', ''), + 'cpu_workers': env.get('MLC_GEEKBENCH_CPU_WORKERS', ''), + 'extra_args': env.get('MLC_GEEKBENCH_EXTRA_ARGS', ''), + }, + 'results': { + 'individual_results': summary.get('individual_results', []), + 'overall_statistics': summary.get('overall_statistics', {}), + 'workload_statistics': summary.get('workload_statistics', {}), + 'runtime_statistics': summary.get('runtime_statistics', {}), + 'individual_runtimes': summary.get('individual_runtimes', []), + }, + 'platform_details': platform_details, + } + + with open(db_json_path, 'w') as f: + json.dump(db_entry, f, indent=2) + + logger.info(f"Benchmark DB entry saved to: {db_json_path}") diff --git a/script/benchmark-program-geekbench/meta.yaml b/script/benchmark-program-geekbench/meta.yaml index 263e7b50c..fb8671f2a 100644 --- a/script/benchmark-program-geekbench/meta.yaml +++ b/script/benchmark-program-geekbench/meta.yaml @@ -3,23 +3,43 @@ automation_alias: script automation_uid: 5b4e0237da074764 category: Benchmarking cache: false - default_env: MLC_GEEKBENCH_WORKLOAD: '' - + MLC_GEEKBENCH_ITERATIONS: '3' + MLC_GEEKBENCH_NUM_RUNS: '1' + MLC_GEEKBENCH_CORE_PINNING: 'no' + MLC_GEEKBENCH_PINNED_CORE: '0' + MLC_GEEKBENCH_EXTRA_ARGS: '' deps: - tags: detect,os - tags: detect,cpu - names: - get-geekbench tags: get,geekbench - +- names: + - get-platform-details + tags: get,platform,details input_mapping: license_email: MLC_GEEKBENCH_LICENSE_EMAIL license_key: MLC_GEEKBENCH_LICENSE_KEY workload: MLC_GEEKBENCH_WORKLOAD - local_file: MLC_GEEKBENCH_LOCAL_FILE - export_json: MLC_GEEKBENCH_EXPORT_JSON + iterations: MLC_GEEKBENCH_ITERATIONS + num_runs: MLC_GEEKBENCH_NUM_RUNS + core_pinning: MLC_GEEKBENCH_CORE_PINNING + pinned_core: MLC_GEEKBENCH_PINNED_CORE + results_dir: MLC_GEEKBENCH_RESULTS_DIR + extra_args: MLC_GEEKBENCH_EXTRA_ARGS + save: MLC_GEEKBENCH_SAVE_FILE + load: MLC_GEEKBENCH_LOAD_FILE + export_html: MLC_GEEKBENCH_EXPORT_HTML + export_xml: MLC_GEEKBENCH_EXPORT_XML + export_text: MLC_GEEKBENCH_EXPORT_TEXT + gpu_platform_id: MLC_GEEKBENCH_GPU_PLATFORM_ID + gpu_device_id: MLC_GEEKBENCH_GPU_DEVICE_ID + section: MLC_GEEKBENCH_SECTION + workload_ids: MLC_GEEKBENCH_WORKLOAD_IDS + cpu_workers: MLC_GEEKBENCH_CPU_WORKERS + workload_gap: MLC_GEEKBENCH_WORKLOAD_GAP new_env_keys: - MLC_GEEKBENCH_* tags: @@ -36,25 +56,45 @@ variations: cpu: default: true env: - MLC_GEEKBENCH_WORKLOAD: '' + MLC_GEEKBENCH_WORKLOAD: '--cpu' + group: workload + gpu: + env: + MLC_GEEKBENCH_WORKLOAD: '--gpu' + group: workload + gpu-vulkan: + env: + MLC_GEEKBENCH_WORKLOAD: '--gpu Vulkan' + group: workload + gpu-opencl: + env: + MLC_GEEKBENCH_WORKLOAD: '--gpu OpenCL' + group: workload + gpu-cuda: + env: + MLC_GEEKBENCH_WORKLOAD: '--gpu CUDA' + group: workload + gpu-metal: + env: + MLC_GEEKBENCH_WORKLOAD: '--gpu Metal' group: workload compute: env: MLC_GEEKBENCH_WORKLOAD: '--compute' group: workload - gpu-opencl: + compute-opencl: env: MLC_GEEKBENCH_WORKLOAD: '--compute OpenCL' group: workload - gpu-vulkan: + compute-vulkan: env: MLC_GEEKBENCH_WORKLOAD: '--compute Vulkan' group: workload - gpu-cuda: + compute-cuda: env: MLC_GEEKBENCH_WORKLOAD: '--compute CUDA' group: workload - gpu-metal: + compute-metal: env: MLC_GEEKBENCH_WORKLOAD: '--compute Metal' group: workload @@ -67,6 +107,31 @@ variations: env: MLC_GEEKBENCH_NO_UPLOAD: 'no' group: upload-mode + single-core: + env: + MLC_GEEKBENCH_SINGLE_CORE: 'yes' + MLC_GEEKBENCH_MULTI_CORE: '' + group: core-mode + multi-core: + env: + MLC_GEEKBENCH_SINGLE_CORE: '' + MLC_GEEKBENCH_MULTI_CORE: 'yes' + group: core-mode + all-cores: + default: true + env: + MLC_GEEKBENCH_SINGLE_CORE: '' + MLC_GEEKBENCH_MULTI_CORE: '' + group: core-mode + sysinfo: + env: + MLC_GEEKBENCH_SYSINFO_ONLY: 'yes' + gpu-list: + env: + MLC_GEEKBENCH_GPU_LIST: 'yes' + workload-list: + env: + MLC_GEEKBENCH_WORKLOAD_LIST: 'yes' versions: 5.5.1: adr: diff --git a/script/benchmark-program-geekbench/run.bat b/script/benchmark-program-geekbench/run.bat index 390f5aa10..5f3dc9246 100644 --- a/script/benchmark-program-geekbench/run.bat +++ b/script/benchmark-program-geekbench/run.bat @@ -22,31 +22,186 @@ if not "!MLC_GEEKBENCH_LICENSE_KEY!" == "" ( ) ) +rem Info-only mode (sysinfo, gpu-list, workload-list, load) +if "%MLC_GEEKBENCH_INFO_ONLY_MODE%" == "yes" ( + echo. + echo Running: %MLC_RUN_CMD% + %MLC_RUN_CMD% + exit /b !ERRORLEVEL! +) + +set NUM_RUNS=%MLC_GEEKBENCH_NUM_RUNS% +set RESULTS_DIR=%MLC_GEEKBENCH_RESULTS_DIR% +set SPLIT_SC_MC=%MLC_GEEKBENCH_SPLIT_SC_MC% +set CORE_PINNING=%MLC_GEEKBENCH_CORE_PINNING% +set AFFINITY_MASK=%MLC_GEEKBENCH_AFFINITY_MASK% + +if "%NUM_RUNS%" == "" set NUM_RUNS=1 +if "%RESULTS_DIR%" == "" set RESULTS_DIR=. +if "%SPLIT_SC_MC%" == "" set SPLIT_SC_MC=no + echo. echo *********************************************************************** -echo Running Geekbench benchmark... -echo Command: !MLC_RUN_CMD! +echo Running Geekbench benchmark +echo Number of runs: %NUM_RUNS% +if "%SPLIT_SC_MC%" == "yes" ( + echo Mode: Split SC ^(pinned^) + MC ^(unpinned^) + echo SC command: %MLC_GEEKBENCH_BASE_CMD_SC% + echo MC command: %MLC_GEEKBENCH_BASE_CMD_MC% +) else ( + echo Base command: %MLC_GEEKBENCH_BASE_CMD% +) echo *********************************************************************** -echo. - -!MLC_RUN_CMD! -set exitstatus=!ERRORLEVEL! -if !exitstatus! NEQ 0 ( +for /L %%R in (1,1,%NUM_RUNS%) do ( echo. - echo Geekbench exited with status: !exitstatus! - exit /b !exitstatus! -) + echo === Run %%R/%NUM_RUNS% === -rem Check if results file was created -if exist "!MLC_GEEKBENCH_RESULTS_FILE!" ( - echo. - echo Geekbench results saved to: !MLC_GEEKBENCH_RESULTS_FILE! - echo. - echo Results summary: - type "!MLC_GEEKBENCH_RESULTS_FILE!" - echo. -) else ( - echo. - echo WARNING: Geekbench results file not found at !MLC_GEEKBENCH_RESULTS_FILE! + rem Record run start time + for /f "tokens=1-4 delims=:." %%a in ("!time!") do ( + set /a "run_start_ms=(((%%a*60)+%%b)*60+%%c)*1000+%%d*10" + ) + + if "!SPLIT_SC_MC!" == "yes" ( + rem --- SPLIT MODE: SC pinned, MC unpinned --- + + rem Single-core run (with core pinning) + set "SC_JSON=%RESULTS_DIR%\geekbench_run%%R_sc.json" + set "SC_CSV=%RESULTS_DIR%\geekbench_run%%R_sc.csv" + set "SC_TIMING=%RESULTS_DIR%\geekbench_run%%R_sc_timing.json" + set "SC_EXPORT=--export-json "!SC_JSON!" --export-csv "!SC_CSV!"" + + echo. + echo --- Single-Core ^(pinned^) --- + + if "%CORE_PINNING%" == "yes" ( + set "SC_FULL=start "" /b /wait /affinity %AFFINITY_MASK% %MLC_GEEKBENCH_BASE_CMD_SC% !SC_EXPORT!" + ) else ( + set "SC_FULL=%MLC_GEEKBENCH_BASE_CMD_SC% !SC_EXPORT!" + ) + echo Command: !SC_FULL! + + for /f "tokens=1-4 delims=:." %%a in ("!time!") do ( + set /a "sc_start_ms=(((%%a*60)+%%b)*60+%%c)*1000+%%d*10" + ) + + !SC_FULL! + set sc_exit=!ERRORLEVEL! + + for /f "tokens=1-4 delims=:." %%a in ("!time!") do ( + set /a "sc_end_ms=(((%%a*60)+%%b)*60+%%c)*1000+%%d*10" + ) + set /a "sc_dur_ms=sc_end_ms-sc_start_ms" + if !sc_dur_ms! LSS 0 set /a "sc_dur_ms+=86400000" + set /a "sc_dur_sec=sc_dur_ms/1000" + set /a "sc_dur_frac=sc_dur_ms%%1000" + + echo {"run": %%R, "phase": "SC", "duration_sec": !sc_dur_sec!.!sc_dur_frac!} > "!SC_TIMING!" + echo SC duration: !sc_dur_sec!.!sc_dur_frac!s + + if !sc_exit! NEQ 0 ( + echo Geekbench single-core exited with status: !sc_exit! ^(run %%R^) + exit /b !sc_exit! + ) + + if exist "!SC_JSON!" ( + echo SC JSON results saved: !SC_JSON! + ) else ( + echo WARNING: SC JSON results not found at !SC_JSON! + ) + + rem Multi-core run (no core pinning) + set "MC_JSON=%RESULTS_DIR%\geekbench_run%%R_mc.json" + set "MC_CSV=%RESULTS_DIR%\geekbench_run%%R_mc.csv" + set "MC_TIMING=%RESULTS_DIR%\geekbench_run%%R_mc_timing.json" + set "MC_EXPORT=--export-json "!MC_JSON!" --export-csv "!MC_CSV!"" + set "MC_FULL=%MLC_GEEKBENCH_BASE_CMD_MC% !MC_EXPORT!" + + echo. + echo --- Multi-Core ^(unpinned^) --- + echo Command: !MC_FULL! + + for /f "tokens=1-4 delims=:." %%a in ("!time!") do ( + set /a "mc_start_ms=(((%%a*60)+%%b)*60+%%c)*1000+%%d*10" + ) + + !MC_FULL! + set mc_exit=!ERRORLEVEL! + + for /f "tokens=1-4 delims=:." %%a in ("!time!") do ( + set /a "mc_end_ms=(((%%a*60)+%%b)*60+%%c)*1000+%%d*10" + ) + set /a "mc_dur_ms=mc_end_ms-mc_start_ms" + if !mc_dur_ms! LSS 0 set /a "mc_dur_ms+=86400000" + set /a "mc_dur_sec=mc_dur_ms/1000" + set /a "mc_dur_frac=mc_dur_ms%%1000" + + echo {"run": %%R, "phase": "MC", "duration_sec": !mc_dur_sec!.!mc_dur_frac!} > "!MC_TIMING!" + echo MC duration: !mc_dur_sec!.!mc_dur_frac!s + + if !mc_exit! NEQ 0 ( + echo Geekbench multi-core exited with status: !mc_exit! ^(run %%R^) + exit /b !mc_exit! + ) + + if exist "!MC_JSON!" ( + echo MC JSON results saved: !MC_JSON! + ) else ( + echo WARNING: MC JSON results not found at !MC_JSON! + ) + + ) else ( + rem --- NORMAL MODE --- + + set "JSON_FILE=%RESULTS_DIR%\geekbench_run%%R.json" + set "CSV_FILE=%RESULTS_DIR%\geekbench_run%%R.csv" + set "EXPORT_ARGS=--export-json "!JSON_FILE!" --export-csv "!CSV_FILE!"" + + if "%CORE_PINNING%" == "yes" ( + set "FULL_CMD=start "" /b /wait /affinity %AFFINITY_MASK% %MLC_GEEKBENCH_BASE_CMD% !EXPORT_ARGS!" + ) else ( + set "FULL_CMD=%MLC_GEEKBENCH_BASE_CMD% !EXPORT_ARGS!" + ) + + echo Command: !FULL_CMD! + + !FULL_CMD! + set exitstatus=!ERRORLEVEL! + + if !exitstatus! NEQ 0 ( + echo. + echo Geekbench exited with status: !exitstatus! ^(run %%R^) + exit /b !exitstatus! + ) + + if exist "!JSON_FILE!" ( + echo JSON results saved: !JSON_FILE! + ) else ( + echo WARNING: JSON results file not found at !JSON_FILE! + ) + + if exist "!CSV_FILE!" ( + echo CSV results saved: !CSV_FILE! + ) + ) + + rem Record total run timing + for /f "tokens=1-4 delims=:." %%a in ("!time!") do ( + set /a "run_end_ms=(((%%a*60)+%%b)*60+%%c)*1000+%%d*10" + ) + set /a "run_dur_ms=run_end_ms-run_start_ms" + if !run_dur_ms! LSS 0 set /a "run_dur_ms+=86400000" + set /a "run_dur_sec=run_dur_ms/1000" + set /a "run_dur_frac=run_dur_ms%%1000" + + set "TIMING_FILE=%RESULTS_DIR%\geekbench_run%%R_timing.json" + echo {"run": %%R, "phase": "total", "duration_sec": !run_dur_sec!.!run_dur_frac!} > "!TIMING_FILE!" + echo Run %%R total duration: !run_dur_sec!.!run_dur_frac!s ) + +echo. +echo *********************************************************************** +echo All %NUM_RUNS% Geekbench run^(s^) completed successfully. +echo *********************************************************************** + +exit /b 0 diff --git a/script/benchmark-program-geekbench/run.sh b/script/benchmark-program-geekbench/run.sh index 7603277ae..e922c4b55 100755 --- a/script/benchmark-program-geekbench/run.sh +++ b/script/benchmark-program-geekbench/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Benchmark Program - Geekbench (Unix) +# Benchmark Program - Geekbench (Unix/Linux) CUR_DIR=$PWD @@ -24,31 +24,149 @@ if [ -n "${MLC_GEEKBENCH_LICENSE_KEY}" ]; then fi fi +# Info-only mode (sysinfo, gpu-list, workload-list, load) +if [ "${MLC_GEEKBENCH_INFO_ONLY_MODE}" == "yes" ]; then + echo "" + echo "Running: ${MLC_RUN_CMD}" + eval ${MLC_RUN_CMD} + exit $? +fi + +NUM_RUNS=${MLC_GEEKBENCH_NUM_RUNS:-1} +RESULTS_DIR=${MLC_GEEKBENCH_RESULTS_DIR:-.} +SPLIT_SC_MC=${MLC_GEEKBENCH_SPLIT_SC_MC:-no} + echo "" echo "***********************************************************************" -echo "Running Geekbench benchmark..." -echo "Command: ${MLC_RUN_CMD}" +echo "Running Geekbench benchmark" +echo " Number of runs: ${NUM_RUNS}" +if [ "${SPLIT_SC_MC}" == "yes" ]; then + echo " Mode: Split SC (pinned) + MC (unpinned)" + echo " SC command: ${MLC_GEEKBENCH_BASE_CMD_SC}" + echo " MC command: ${MLC_GEEKBENCH_BASE_CMD_MC}" +else + echo " Base command: ${MLC_GEEKBENCH_BASE_CMD}" +fi echo "***********************************************************************" -echo "" - -eval ${MLC_RUN_CMD} -exitstatus=$? -if [ ${exitstatus} -ne 0 ]; then +for run in $(seq 1 ${NUM_RUNS}); do echo "" - echo "Geekbench exited with status: ${exitstatus}" - exit ${exitstatus} -fi + echo "=== Run ${run}/${NUM_RUNS} ===" -# Check if results file was created -if [ -f "${MLC_GEEKBENCH_RESULTS_FILE}" ]; then - echo "" - echo "Geekbench results saved to: ${MLC_GEEKBENCH_RESULTS_FILE}" - echo "" - echo "Results summary:" - cat "${MLC_GEEKBENCH_RESULTS_FILE}" - echo "" -else - echo "" - echo "WARNING: Geekbench results file not found at ${MLC_GEEKBENCH_RESULTS_FILE}" -fi + # Record overall run start time + run_start_epoch=$(date +%s%3N) + run_start_ts=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) + + if [ "${SPLIT_SC_MC}" == "yes" ]; then + # --- SPLIT MODE: Run single-core (pinned) then multi-core (unpinned) --- + + # Single-core run + SC_JSON="${RESULTS_DIR}/geekbench_run${run}_sc.json" + SC_CSV="${RESULTS_DIR}/geekbench_run${run}_sc.csv" + SC_TIMING="${RESULTS_DIR}/geekbench_run${run}_sc_timing.json" + SC_EXPORT=" --export-json '${SC_JSON}' --export-csv '${SC_CSV}'" + SC_FULL="${MLC_GEEKBENCH_BASE_CMD_SC}${SC_EXPORT}" + + echo "" + echo "--- Single-Core (pinned) ---" + echo "Command: ${SC_FULL}" + + sc_start=$(date +%s%3N) + eval ${SC_FULL} + sc_exit=$? + sc_end=$(date +%s%3N) + sc_dur_ms=$((sc_end - sc_start)) + sc_dur=$(echo "scale=3; ${sc_dur_ms} / 1000" | bc) + + echo "{\"run\": ${run}, \"phase\": \"SC\", \"duration_sec\": ${sc_dur}}" > "${SC_TIMING}" + echo "SC duration: ${sc_dur}s" + + if [ ${sc_exit} -ne 0 ]; then + echo "Geekbench single-core exited with status: ${sc_exit} (run ${run})" + exit ${sc_exit} + fi + + if [ -f "${SC_JSON}" ]; then + echo "SC JSON results saved: ${SC_JSON}" + else + echo "WARNING: SC JSON results not found at ${SC_JSON}" + fi + + # Multi-core run + MC_JSON="${RESULTS_DIR}/geekbench_run${run}_mc.json" + MC_CSV="${RESULTS_DIR}/geekbench_run${run}_mc.csv" + MC_TIMING="${RESULTS_DIR}/geekbench_run${run}_mc_timing.json" + MC_EXPORT=" --export-json '${MC_JSON}' --export-csv '${MC_CSV}'" + MC_FULL="${MLC_GEEKBENCH_BASE_CMD_MC}${MC_EXPORT}" + + echo "" + echo "--- Multi-Core (unpinned) ---" + echo "Command: ${MC_FULL}" + + mc_start=$(date +%s%3N) + eval ${MC_FULL} + mc_exit=$? + mc_end=$(date +%s%3N) + mc_dur_ms=$((mc_end - mc_start)) + mc_dur=$(echo "scale=3; ${mc_dur_ms} / 1000" | bc) + + echo "{\"run\": ${run}, \"phase\": \"MC\", \"duration_sec\": ${mc_dur}}" > "${MC_TIMING}" + echo "MC duration: ${mc_dur}s" + + if [ ${mc_exit} -ne 0 ]; then + echo "Geekbench multi-core exited with status: ${mc_exit} (run ${run})" + exit ${mc_exit} + fi + + if [ -f "${MC_JSON}" ]; then + echo "MC JSON results saved: ${MC_JSON}" + else + echo "WARNING: MC JSON results not found at ${MC_JSON}" + fi + + else + # --- NORMAL MODE: Single command --- + + JSON_FILE="${RESULTS_DIR}/geekbench_run${run}.json" + CSV_FILE="${RESULTS_DIR}/geekbench_run${run}.csv" + EXPORT_ARGS=" --export-json '${JSON_FILE}' --export-csv '${CSV_FILE}'" + FULL_CMD="${MLC_GEEKBENCH_BASE_CMD}${EXPORT_ARGS}" + + echo "Command: ${FULL_CMD}" + + eval ${FULL_CMD} + exitstatus=$? + + if [ ${exitstatus} -ne 0 ]; then + echo "" + echo "Geekbench exited with status: ${exitstatus} (run ${run})" + exit ${exitstatus} + fi + + if [ -f "${JSON_FILE}" ]; then + echo "JSON results saved: ${JSON_FILE}" + else + echo "WARNING: JSON results file not found at ${JSON_FILE}" + fi + + if [ -f "${CSV_FILE}" ]; then + echo "CSV results saved: ${CSV_FILE}" + fi + fi + + # Record overall run timing + run_end_epoch=$(date +%s%3N) + run_end_ts=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) + run_dur_ms=$((run_end_epoch - run_start_epoch)) + run_dur=$(echo "scale=3; ${run_dur_ms} / 1000" | bc) + + TIMING_FILE="${RESULTS_DIR}/geekbench_run${run}_timing.json" + echo "{\"run\": ${run}, \"phase\": \"total\", \"start\": \"${run_start_ts}\", \"end\": \"${run_end_ts}\", \"duration_sec\": ${run_dur}}" > "${TIMING_FILE}" + echo "Run ${run} total duration: ${run_dur}s" + +done + +echo "" +echo "***********************************************************************" +echo "All ${NUM_RUNS} Geekbench run(s) completed successfully." +echo "***********************************************************************" From d4f37495a0894816630b71bd0aa2b47a4c7da17f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 18 Mar 2026 20:52:30 +0000 Subject: [PATCH 44/53] [Automated Commit] Document script/benchmark-program-geekbench/meta.yaml [skip ci] --- script/benchmark-program-geekbench/README.md | 36 ++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/script/benchmark-program-geekbench/README.md b/script/benchmark-program-geekbench/README.md index f78dee770..603087e81 100644 --- a/script/benchmark-program-geekbench/README.md +++ b/script/benchmark-program-geekbench/README.md @@ -43,8 +43,23 @@ mlcr benchmark,geekbench,benchmark-geekbench,benchmark-program-geekbench | `--license_email` | | | `` | | `--license_key` | | | `` | | `--workload` | | | `` | -| `--local_file` | | | `` | -| `--export_json` | | | `` | +| `--iterations` | | | `3` | +| `--num_runs` | | | `1` | +| `--core_pinning` | | | `no` | +| `--pinned_core` | | | `0` | +| `--results_dir` | | | `` | +| `--extra_args` | | | `` | +| `--save` | | | `` | +| `--load` | | | `` | +| `--export_html` | | | `` | +| `--export_xml` | | | `` | +| `--export_text` | | | `` | +| `--gpu_platform_id` | | | `` | +| `--gpu_device_id` | | | `` | +| `--section` | | | `` | +| `--workload_ids` | | | `` | +| `--cpu_workers` | | | `` | +| `--workload_gap` | | | `` | ### Generic Script Inputs | Name | Description | Choices | Default | @@ -67,6 +82,18 @@ mlcr benchmark,geekbench,benchmark-geekbench,benchmark-program-geekbench | `--verify_ssl` | Verify SSL | | `False` | ## Variations +### Core-mode + +- `all-cores` (default) +- `multi-core` +- `single-core` + +### Ungrouped + +- `gpu-list` +- `sysinfo` +- `workload-list` + ### Upload-mode - `no-upload` (default) @@ -75,7 +102,12 @@ mlcr benchmark,geekbench,benchmark-geekbench,benchmark-program-geekbench ### Workload - `compute` +- `compute-cuda` +- `compute-metal` +- `compute-opencl` +- `compute-vulkan` - `cpu` (default) +- `gpu` - `gpu-cuda` - `gpu-metal` - `gpu-opencl` From 7d898db03a097ef8c34677016857c60727c36fc3 Mon Sep 17 00:00:00 2001 From: amd-arsuresh Date: Thu, 19 Mar 2026 19:35:34 +0000 Subject: [PATCH 45/53] Add structured platform details, improved geekbench-benchmark (#867) * Add structured platform details * Make get-platform-details script generic * Dont use sudo for platform-details on macos * Changes to support GB result generation * Fix reuse_logs for geekbench * Fixes for GB dont export to json without license * Fixes for get-platform-details on windows --- README.md | 2 +- script/README.md | 2 +- .../benchmark-program-geekbench/customize.py | 197 ++-- script/benchmark-program-geekbench/meta.yaml | 3 +- script/benchmark-program-geekbench/run.bat | 11 +- script/benchmark-program-geekbench/run.sh | 30 +- script/get-platform-details/customize.py | 963 +++++++++++++++++- script/get-platform-details/meta.yaml | 6 + script/get-platform-details/run-macos.sh | 97 +- script/get-platform-details/run.bat | 83 +- script/get-platform-details/run.sh | 31 +- 11 files changed, 1272 insertions(+), 153 deletions(-) diff --git a/README.md b/README.md index 2ecd4ac2d..408c57148 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,11 @@ [![MLPerf Inference ABTF POC Test](https://github.com/mlcommons/mlperf-automations/actions/workflows/test-mlperf-inference-abtf-poc.yml/badge.svg)](https://github.com/mlcommons/mlperf-automations/actions/workflows/test-mlperf-inference-abtf-poc.yml) + Welcome to the **MLPerf Automations and Scripts** repository! This repository is your go-to resource for tools, automations, and scripts designed to streamline the execution of **MLPerf benchmarks**—with a strong emphasis on **MLPerf Inference benchmarks**. Starting **January 2025**, MLPerf automation scripts is powered by [MLCFlow](https://github.com/mlcommons/mlcflow) automation interface. This new and simplified framework replaces the previous [Collective Mind (CM)](https://github.com/mlcommons/ck/tree/master/cm), providing a more robust, efficient, and self-contained solution for benchmarking workflows, making MLPerf automations independent of any external projects. - --- ## 🚀 Key Features diff --git a/script/README.md b/script/README.md index 0ae1e3a97..91780a074 100644 --- a/script/README.md +++ b/script/README.md @@ -1,6 +1,6 @@ # MLCommons Automation Scripts -*Last updated: 2026-03-19 02:02:58* +*Last updated: 2026-03-19 20:21:31* This directory contains automation scripts for MLPerf benchmarks, AI/ML workflows, and development operations. diff --git a/script/benchmark-program-geekbench/customize.py b/script/benchmark-program-geekbench/customize.py index 52cde1ce6..186ee28c9 100644 --- a/script/benchmark-program-geekbench/customize.py +++ b/script/benchmark-program-geekbench/customize.py @@ -48,7 +48,7 @@ def extract_scores(result): def extract_workload_scores(result): """Extract per-section and per-workload scores from a geekbench result dict. Returns a dict keyed by section name, each containing section score and - a dict of workload name -> {score, runtime, runtimes, runtime_warmup}. + per-workload details including per-iteration runtimes. Geekbench's --iterations N flag results in 1 warmup iteration + (N-1) scored iterations. The 'runtimes' list contains per-scored-iteration @@ -94,21 +94,21 @@ def merge_sc_mc_results(sc_result, mc_result): def compute_stats(values): - """Compute mean, median, olympic, stdev, min, max for a list of numeric values.""" + """Compute mean, median, olympic, cv_percent, min, max for a list of numeric values.""" if not values: return {} + mean_val = round(statistics.mean(values), 2) + stdev_val = round(statistics.stdev(values), 2) if len(values) >= 2 else 0 result = { - 'mean': round(statistics.mean(values), 2), + 'mean': mean_val, 'median': round(statistics.median(values), 2), 'olympic': round(compute_olympic_score(values), 2), - 'stdev': round(statistics.stdev(values), 2) if len(values) >= 2 else 0, 'min': round(min(values), 2), 'max': round(max(values), 2), 'count': len(values) } - if result['mean'] != 0: - result['cv_percent'] = round( - (result['stdev'] / abs(result['mean'])) * 100, 2) + if mean_val != 0: + result['cv_percent'] = round((stdev_val / abs(mean_val)) * 100, 2) else: result['cv_percent'] = 0 return result @@ -141,6 +141,14 @@ def preprocess(i): env = i['env'] logger = i['automation'].logger + # Check if platform details should be skipped + skip_platform_details = is_true( + env.get('MLC_GEEKBENCH_SKIP_PLATFORM_DETAILS', 'no')) + if skip_platform_details: + logger.info( + "Platform details collection will be skipped (skip_platform_details=yes)") + env['MLC_SKIP_DEPENDENCY_get-platform-details'] = 'yes' + geekbench_bin = env.get('MLC_GEEKBENCH_BIN_WITH_PATH', '') if geekbench_bin == '' or not os.path.isfile(geekbench_bin): return {'return': 1, @@ -189,6 +197,13 @@ def preprocess(i): env['MLC_GEEKBENCH_INFO_ONLY_MODE'] = 'yes' return {'return': 0} + # --- Reuse logs mode (skip execution, just re-run postprocess) --- + if is_true(env.get('MLC_GEEKBENCH_REUSE_LOGS', '')): + env['MLC_RUN_CMD'] = ' ' + logger.info( + "Reuse logs mode: skipping benchmark execution, will parse existing results") + return {'return': 0} + # --- Build the base benchmark command --- args = [] @@ -223,8 +238,10 @@ def preprocess(i): args.append(f'--workload {workload_ids}') # Iterations (native geekbench: --iterations N) - iterations = env.get('MLC_GEEKBENCH_ITERATIONS', '3').strip() - args.append(f'--iterations {iterations}') + # Only pass --iterations if explicitly set; otherwise use geekbench default + iterations = env.get('MLC_GEEKBENCH_ITERATIONS', '').strip() + if iterations: + args.append(f'--iterations {iterations}') # Workload gap (Pro: --workload-gap N milliseconds) workload_gap = env.get('MLC_GEEKBENCH_WORKLOAD_GAP', '').strip() @@ -281,11 +298,6 @@ def preprocess(i): platform = os_info['platform'] # --- Split SC/MC mode: core_pinning + all-cores --- - # When core pinning is enabled and neither --single-core nor --multi-core - # was explicitly chosen, we split into two invocations per run: - # 1) Single-core with core pinning - # 2) Multi-core without core pinning - # The shell scripts handle the split and postprocess merges the results. if core_pinning and all_cores_mode: env['MLC_GEEKBENCH_SPLIT_SC_MC'] = 'yes' @@ -316,8 +328,7 @@ def preprocess(i): env['MLC_GEEKBENCH_SPLIT_SC_MC'] = 'no' base_cmd = f"{q}{geekbench_bin}{q} {' '.join(args)}{extra_exports}" - # Apply core pinning to the single command (only sensible for - # --single-core) + # Apply core pinning to the single command if core_pinning: if explicit_multi_core: logger.warning("Core pinning with --multi-core will pin all threads to one core. " @@ -337,8 +348,9 @@ def preprocess(i): env['MLC_GEEKBENCH_BASE_CMD'] = base_cmd logger.info(f"Geekbench command: {base_cmd}") + iter_display = iterations if iterations else 'default (geekbench built-in)' logger.info( - f"Native iterations per workload: {iterations}, Number of runs: {num_runs}") + f"Native iterations per workload: {iter_display}, Number of runs: {num_runs}") logger.info(f"Results directory: {results_dir}") return {'return': 0} @@ -357,7 +369,7 @@ def postprocess(i): results_dir = env.get('MLC_GEEKBENCH_RESULTS_DIR', '') num_runs = int(env.get('MLC_GEEKBENCH_NUM_RUNS', '1')) - iterations_cfg = env.get('MLC_GEEKBENCH_ITERATIONS', '3') + iterations_cfg = env.get('MLC_GEEKBENCH_ITERATIONS', 'default') split_mode = is_true(env.get('MLC_GEEKBENCH_SPLIT_SC_MC', 'no')) all_results = [] @@ -534,7 +546,6 @@ def postprocess(i): } # --- Runtime statistics --- - # For split mode, compute separate SC/MC/total stats runtime_stats = {} total_durations = [d['duration_sec'] for d in run_durations if d['phase'] == 'total'] @@ -572,17 +583,34 @@ def postprocess(i): 'individual_runtimes': run_durations, } + # Add platform details if available and not skipped + skip_platform_details = is_true( + env.get('MLC_GEEKBENCH_SKIP_PLATFORM_DETAILS', 'no')) + if not skip_platform_details: + platform_details_file = env.get('MLC_PLATFORM_DETAILS_FILE_PATH', '') + if platform_details_file and os.path.isfile(platform_details_file): + try: + with open(platform_details_file, 'r') as f: + platform_details = json.load(f) + summary['platform_details'] = platform_details + logger.info( + f"Platform details loaded from: {platform_details_file}") + except Exception as e: + logger.warning(f"Failed to load platform details: {e}") + else: + logger.info( + "Platform details collection skipped (skip_platform_details=yes)") + # Log key results for key, stat in overall_stats.items(): logger.info( f"Overall {key}: mean={stat['mean']}, " f"median={stat['median']}, olympic={stat['olympic']}, " - f"stdev={stat['stdev']}, cv={stat['cv_percent']}%") + f"cv={stat['cv_percent']}%") for phase, rstats in runtime_stats.items(): logger.info( f"Runtime {phase} (sec): mean={rstats['mean']}, " - f"stdev={rstats['stdev']}, " f"cv={rstats['cv_percent']}%, " f"min={rstats['min']}, max={rstats['max']}") @@ -612,11 +640,6 @@ def postprocess(i): # --- Print results in tabular format --- _print_results_table(summary) - # --- Create benchmark database entry JSON --- - db_json_path = os.path.join(results_dir, 'geekbench_benchmark_entry.json') - _write_benchmark_db_entry(env, summary, db_json_path, logger) - env['MLC_GEEKBENCH_BENCHMARK_ENTRY_JSON'] = db_json_path - state['geekbench_results'] = all_results state['geekbench_summary'] = summary @@ -645,6 +668,12 @@ def _print_results_table(summary): elif d['phase'] == 'MC': mc_dur[run] = d['duration_sec'] + # Print CV description + print("") + print(" Note: CV% (Coefficient of Variation) = (StdDev / Mean) * 100.") + print(" Lower CV% indicates more consistent/reproducible results.") + print(" CV% < 1% is excellent, 1-5% is good, > 5% may need investigation.") + # ============================================================ # 1. Individual Run Top-Level Scores # ============================================================ @@ -709,7 +738,7 @@ def _print_results_table(summary): for it in range(1, max_iters + 1): headers.append(f'Iter {it} (s)') if max_iters >= 2: - headers.append('Iter Stdev') + headers.append('Iter CV%') rows = [] for wl_name, wl_info in sec_info['workloads'].items(): @@ -727,7 +756,12 @@ def _print_results_table(summary): row.append('-') if max_iters >= 2: if len(iters) >= 2: - row.append(round(statistics.stdev(iters), 6)) + mean_rt = statistics.mean(iters) + std_rt = statistics.stdev(iters) + cv_pct = round( + (std_rt / abs(mean_rt)) * 100, + 2) if mean_rt != 0 else 0 + row.append(cv_pct) else: row.append('-') rows.append(row) @@ -745,7 +779,6 @@ def _print_results_table(summary): 'Mean', 'Median', 'Olympic', - 'Stdev', 'CV%', 'Min', 'Max', @@ -757,7 +790,6 @@ def _print_results_table(summary): stats.get('mean', '-'), stats.get('median', '-'), stats.get('olympic', '-'), - stats.get('stdev', '-'), stats.get('cv_percent', '-'), stats.get('min', '-'), stats.get('max', '-'), @@ -769,7 +801,6 @@ def _print_results_table(summary): rstats.get('mean', '-'), rstats.get('median', '-'), rstats.get('olympic', '-'), - rstats.get('stdev', '-'), rstats.get('cv_percent', '-'), rstats.get('min', '-'), rstats.get('max', '-'), @@ -800,13 +831,12 @@ def _print_results_table(summary): print("=" * 78) print(f" WORKLOAD STATISTICS (across runs) | {sec_name}{pinned_note}" f" (section score: mean={sec_stats.get('mean', '-')}," - f" stdev={sec_stats.get('stdev', '-')}," f" cv={sec_stats.get('cv_percent', '-')}%)") print("=" * 78) headers = ['Workload', - 'Score Mean', 'Score Stdev', 'Score CV%', - 'RT Mean', 'RT Stdev', 'RT CV%', - 'Iter RT Mean', 'Iter RT Stdev', 'Iter RT CV%'] + 'Score Mean', 'Score CV%', + 'RT Mean', 'RT CV%', + 'Iter RT Mean', 'Iter RT CV%'] rows = [] for wl_name, wl_stats in sec_info['workloads'].items(): s = wl_stats.get('score', {}) @@ -814,12 +844,9 @@ def _print_results_table(summary): ir = wl_stats.get('iteration_runtimes', {}) rows.append([ wl_name, - s.get('mean', '-'), s.get('stdev', - '-'), s.get('cv_percent', '-'), - r.get('mean', '-'), r.get('stdev', - '-'), r.get('cv_percent', '-'), - ir.get('mean', '-'), ir.get('stdev', - '-'), ir.get('cv_percent', '-'), + s.get('mean', '-'), s.get('cv_percent', '-'), + r.get('mean', '-'), r.get('cv_percent', '-'), + ir.get('mean', '-'), ir.get('cv_percent', '-'), ]) _print_table(headers, rows) @@ -835,11 +862,13 @@ def _print_results_table(summary): pinned_note = " [UNPINNED]" print("") print("=" * 78) - print(f" WORKLOAD RESULTS | {sec_name}{pinned_note}" + print(f" ITERATION VARIANCE | {sec_name}{pinned_note}" f" (section score: {sec_stats.get('mean', '-')})") + print(" (Variance is between iterations within a single run." + " Geekbench reports per-iteration runtimes only, not scores.)") print("=" * 78) headers = ['Workload', 'Score', 'Mean RT (s)', - 'Iter RT Stdev', 'Iter RT CV%', 'Iters Scored'] + 'Iter RT CV%', 'Iters Scored'] rows = [] for wl_name, wl_stats in sec_info['workloads'].items(): s = wl_stats.get('score', {}) @@ -849,7 +878,6 @@ def _print_results_table(summary): wl_name, s.get('mean', '-'), r.get('mean', '-'), - ir.get('stdev', '-'), ir.get('cv_percent', '-'), ir.get('count', '-'), ]) @@ -919,7 +947,7 @@ def _write_summary_csv(summary, csv_path): for it in range(1, max_iters + 1): header.append(f'iter_{it}_runtime') if max_iters >= 2: - header.append('iter_stdev') + header.append('iter_cv_percent') writer.writerow(header) for wl_name, wl_info in sec_info['workloads'].items(): iters = wl_info.get('runtimes', []) @@ -933,7 +961,12 @@ def _write_summary_csv(summary, csv_path): 6) if it < len(iters) else '') if max_iters >= 2: if len(iters) >= 2: - row.append(round(statistics.stdev(iters), 6)) + mean_rt = statistics.mean(iters) + std_rt = statistics.stdev(iters) + cv_pct = round( + (std_rt / abs(mean_rt)) * 100, + 2) if mean_rt != 0 else 0 + row.append(cv_pct) else: row.append('') writer.writerow(row) @@ -942,10 +975,10 @@ def _write_summary_csv(summary, csv_path): # --- Overall statistics --- writer.writerow(['Overall Statistics']) writer.writerow(['metric', 'mean', 'median', 'olympic', - 'stdev', 'cv_percent', 'min', 'max', 'count']) + 'cv_percent', 'min', 'max', 'count']) for metric, stats in summary.get('overall_statistics', {}).items(): writer.writerow([metric, stats.get('mean', ''), stats.get('median', ''), - stats.get('olympic', ''), stats.get('stdev', ''), + stats.get('olympic', ''), stats.get('cv_percent', ''), stats.get('min', ''), stats.get('max', ''), stats.get('count', '')]) for phase, rstats in summary.get('runtime_statistics', {}).items(): @@ -953,9 +986,7 @@ def _write_summary_csv(summary, csv_path): rstats.get( 'median', ''), rstats.get( 'olympic', ''), - rstats.get( - 'stdev', ''), rstats.get( - 'cv_percent', ''), + rstats.get('cv_percent', ''), rstats.get('min', ''), rstats.get('max', ''), rstats.get('count', '')]) writer.writerow([]) @@ -969,10 +1000,10 @@ def _write_summary_csv(summary, csv_path): [f'Workload Statistics | {sec_name} (section score mean: {sec_ss.get("mean", "")})']) writer.writerow(['workload', 'score_mean', 'score_median', 'score_olympic', - 'score_stdev', 'score_cv_percent', + 'score_cv_percent', 'runtime_mean', 'runtime_median', 'runtime_olympic', - 'runtime_stdev', 'runtime_cv_percent', - 'iter_rt_mean', 'iter_rt_stdev', 'iter_rt_cv_percent']) + 'runtime_cv_percent', + 'iter_rt_mean', 'iter_rt_cv_percent']) for wl_name, wl_stats in sec_info['workloads'].items(): s = wl_stats.get('score', {}) r = wl_stats.get('runtime', {}) @@ -983,64 +1014,12 @@ def _write_summary_csv(summary, csv_path): 'mean', ''), s.get( 'median', ''), s.get( 'olympic', ''), - s.get('stdev', ''), s.get('cv_percent', ''), + s.get('cv_percent', ''), r.get( 'mean', ''), r.get( 'median', ''), r.get( 'olympic', ''), - r.get('stdev', ''), r.get('cv_percent', ''), - ir.get( - 'mean', ''), ir.get( - 'stdev', ''), ir.get( - 'cv_percent', ''), + r.get('cv_percent', ''), + ir.get('mean', ''), ir.get('cv_percent', ''), ]) writer.writerow([]) - - -def _write_benchmark_db_entry(env, summary, db_json_path, logger): - """Create a JSON file combining benchmark results with platform details.""" - - platform_details = {} - platform_details_file = env.get('MLC_PLATFORM_DETAILS_FILE_PATH', '') - if platform_details_file and os.path.isfile(platform_details_file): - try: - with open(platform_details_file, 'r') as f: - platform_details = json.load(f) - logger.info( - f"Loaded platform details from: {platform_details_file}") - except Exception as e: - logger.warning(f"Could not load platform details: {e}") - else: - logger.warning( - "Platform details file not found. " - "Benchmark DB entry will not include platform information.") - - db_entry = { - 'benchmark': 'geekbench', - 'timestamp': datetime.datetime.now(datetime.timezone.utc).isoformat(), - 'configuration': { - 'workload': env.get('MLC_GEEKBENCH_WORKLOAD', ''), - 'iterations': env.get('MLC_GEEKBENCH_ITERATIONS', '3'), - 'num_runs': env.get('MLC_GEEKBENCH_NUM_RUNS', '1'), - 'core_pinning': env.get('MLC_GEEKBENCH_CORE_PINNING', 'no'), - 'pinned_core': env.get('MLC_GEEKBENCH_PINNED_CORE', '0'), - 'split_sc_mc': env.get('MLC_GEEKBENCH_SPLIT_SC_MC', 'no'), - 'single_core': env.get('MLC_GEEKBENCH_SINGLE_CORE', ''), - 'multi_core': env.get('MLC_GEEKBENCH_MULTI_CORE', ''), - 'cpu_workers': env.get('MLC_GEEKBENCH_CPU_WORKERS', ''), - 'extra_args': env.get('MLC_GEEKBENCH_EXTRA_ARGS', ''), - }, - 'results': { - 'individual_results': summary.get('individual_results', []), - 'overall_statistics': summary.get('overall_statistics', {}), - 'workload_statistics': summary.get('workload_statistics', {}), - 'runtime_statistics': summary.get('runtime_statistics', {}), - 'individual_runtimes': summary.get('individual_runtimes', []), - }, - 'platform_details': platform_details, - } - - with open(db_json_path, 'w') as f: - json.dump(db_entry, f, indent=2) - - logger.info(f"Benchmark DB entry saved to: {db_json_path}") diff --git a/script/benchmark-program-geekbench/meta.yaml b/script/benchmark-program-geekbench/meta.yaml index fb8671f2a..91f7d1dc4 100644 --- a/script/benchmark-program-geekbench/meta.yaml +++ b/script/benchmark-program-geekbench/meta.yaml @@ -5,7 +5,6 @@ category: Benchmarking cache: false default_env: MLC_GEEKBENCH_WORKLOAD: '' - MLC_GEEKBENCH_ITERATIONS: '3' MLC_GEEKBENCH_NUM_RUNS: '1' MLC_GEEKBENCH_CORE_PINNING: 'no' MLC_GEEKBENCH_PINNED_CORE: '0' @@ -40,6 +39,8 @@ input_mapping: workload_ids: MLC_GEEKBENCH_WORKLOAD_IDS cpu_workers: MLC_GEEKBENCH_CPU_WORKERS workload_gap: MLC_GEEKBENCH_WORKLOAD_GAP + reuse_logs: MLC_GEEKBENCH_REUSE_LOGS + skip_platform_details: MLC_GEEKBENCH_SKIP_PLATFORM_DETAILS new_env_keys: - MLC_GEEKBENCH_* tags: diff --git a/script/benchmark-program-geekbench/run.bat b/script/benchmark-program-geekbench/run.bat index 5f3dc9246..469144e31 100644 --- a/script/benchmark-program-geekbench/run.bat +++ b/script/benchmark-program-geekbench/run.bat @@ -40,6 +40,8 @@ if "%NUM_RUNS%" == "" set NUM_RUNS=1 if "%RESULTS_DIR%" == "" set RESULTS_DIR=. if "%SPLIT_SC_MC%" == "" set SPLIT_SC_MC=no +set "HAS_LICENSE=no" +if defined MLC_GEEKBENCH_LICENSE_KEY set "HAS_LICENSE=yes" echo. echo *********************************************************************** echo Running Geekbench benchmark @@ -69,7 +71,8 @@ for /L %%R in (1,1,%NUM_RUNS%) do ( set "SC_JSON=%RESULTS_DIR%\geekbench_run%%R_sc.json" set "SC_CSV=%RESULTS_DIR%\geekbench_run%%R_sc.csv" set "SC_TIMING=%RESULTS_DIR%\geekbench_run%%R_sc_timing.json" - set "SC_EXPORT=--export-json "!SC_JSON!" --export-csv "!SC_CSV!"" + set "SC_EXPORT=" + if "!HAS_LICENSE!"=="yes" set "SC_EXPORT=--export-json "!SC_JSON!" --export-csv "!SC_CSV!" echo. echo --- Single-Core ^(pinned^) --- @@ -114,7 +117,8 @@ for /L %%R in (1,1,%NUM_RUNS%) do ( set "MC_JSON=%RESULTS_DIR%\geekbench_run%%R_mc.json" set "MC_CSV=%RESULTS_DIR%\geekbench_run%%R_mc.csv" set "MC_TIMING=%RESULTS_DIR%\geekbench_run%%R_mc_timing.json" - set "MC_EXPORT=--export-json "!MC_JSON!" --export-csv "!MC_CSV!"" + set "MC_EXPORT=" + if "!HAS_LICENSE!"=="yes" set "MC_EXPORT=--export-json "!MC_JSON!" --export-csv "!MC_CSV!" set "MC_FULL=%MLC_GEEKBENCH_BASE_CMD_MC% !MC_EXPORT!" echo. @@ -155,7 +159,8 @@ for /L %%R in (1,1,%NUM_RUNS%) do ( set "JSON_FILE=%RESULTS_DIR%\geekbench_run%%R.json" set "CSV_FILE=%RESULTS_DIR%\geekbench_run%%R.csv" - set "EXPORT_ARGS=--export-json "!JSON_FILE!" --export-csv "!CSV_FILE!"" + set "EXPORT_ARGS=" + if "!HAS_LICENSE!"=="yes" set "EXPORT_ARGS=--export-json "!JSON_FILE!" --export-csv "!CSV_FILE!" if "%CORE_PINNING%" == "yes" ( set "FULL_CMD=start "" /b /wait /affinity %AFFINITY_MASK% %MLC_GEEKBENCH_BASE_CMD% !EXPORT_ARGS!" diff --git a/script/benchmark-program-geekbench/run.sh b/script/benchmark-program-geekbench/run.sh index e922c4b55..c8347544e 100755 --- a/script/benchmark-program-geekbench/run.sh +++ b/script/benchmark-program-geekbench/run.sh @@ -32,10 +32,25 @@ if [ "${MLC_GEEKBENCH_INFO_ONLY_MODE}" == "yes" ]; then exit $? fi + +# Reuse logs mode: skip benchmark execution, postprocess will parse existing results +if [ "${MLC_GEEKBENCH_REUSE_LOGS}" == "yes" ] || [ "${MLC_GEEKBENCH_REUSE_LOGS}" == "True" ] || [ "${MLC_GEEKBENCH_REUSE_LOGS}" == "on" ] || [ "${MLC_GEEKBENCH_REUSE_LOGS}" == "1" ]; then + echo "" + echo "Reuse logs mode: skipping benchmark execution" + exit 0 +fi + NUM_RUNS=${MLC_GEEKBENCH_NUM_RUNS:-1} RESULTS_DIR=${MLC_GEEKBENCH_RESULTS_DIR:-.} SPLIT_SC_MC=${MLC_GEEKBENCH_SPLIT_SC_MC:-no} +# --export-json/csv require a license +if [ -n "${MLC_GEEKBENCH_LICENSE_KEY}" ]; then + HAS_LICENSE=yes +else + HAS_LICENSE=no +fi + echo "" echo "***********************************************************************" echo "Running Geekbench benchmark" @@ -64,7 +79,10 @@ for run in $(seq 1 ${NUM_RUNS}); do SC_JSON="${RESULTS_DIR}/geekbench_run${run}_sc.json" SC_CSV="${RESULTS_DIR}/geekbench_run${run}_sc.csv" SC_TIMING="${RESULTS_DIR}/geekbench_run${run}_sc_timing.json" - SC_EXPORT=" --export-json '${SC_JSON}' --export-csv '${SC_CSV}'" + SC_EXPORT="" + if [ "${HAS_LICENSE}" == "yes" ]; then + SC_EXPORT=" --export-json '${SC_JSON}' --export-csv '${SC_CSV}'" + fi SC_FULL="${MLC_GEEKBENCH_BASE_CMD_SC}${SC_EXPORT}" echo "" @@ -96,7 +114,10 @@ for run in $(seq 1 ${NUM_RUNS}); do MC_JSON="${RESULTS_DIR}/geekbench_run${run}_mc.json" MC_CSV="${RESULTS_DIR}/geekbench_run${run}_mc.csv" MC_TIMING="${RESULTS_DIR}/geekbench_run${run}_mc_timing.json" - MC_EXPORT=" --export-json '${MC_JSON}' --export-csv '${MC_CSV}'" + MC_EXPORT="" + if [ "${HAS_LICENSE}" == "yes" ]; then + MC_EXPORT=" --export-json '${MC_JSON}' --export-csv '${MC_CSV}'" + fi MC_FULL="${MLC_GEEKBENCH_BASE_CMD_MC}${MC_EXPORT}" echo "" @@ -129,7 +150,10 @@ for run in $(seq 1 ${NUM_RUNS}); do JSON_FILE="${RESULTS_DIR}/geekbench_run${run}.json" CSV_FILE="${RESULTS_DIR}/geekbench_run${run}.csv" - EXPORT_ARGS=" --export-json '${JSON_FILE}' --export-csv '${CSV_FILE}'" + EXPORT_ARGS="" + if [ "${HAS_LICENSE}" == "yes" ]; then + EXPORT_ARGS=" --export-json '${JSON_FILE}' --export-csv '${CSV_FILE}'" + fi FULL_CMD="${MLC_GEEKBENCH_BASE_CMD}${EXPORT_ARGS}" echo "Command: ${FULL_CMD}" diff --git a/script/get-platform-details/customize.py b/script/get-platform-details/customize.py index e13e3864f..a530cdf73 100644 --- a/script/get-platform-details/customize.py +++ b/script/get-platform-details/customize.py @@ -1,5 +1,7 @@ from mlc import utils import os +import json +import re import subprocess @@ -9,7 +11,7 @@ def check_installation(command, os_info): [command, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) == 0 elif os_info['platform'] == "linux": return subprocess.call(['which', command], stdout=subprocess.PIPE, - stderr=subprocess.PIPE) == 0 # 0 means the package is there + stderr=subprocess.PIPE) == 0 def preprocess(i): @@ -17,11 +19,10 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] - if not check_installation("numactl", os_info): - env['MLC_INSTALL_NUMACTL'] = 'True' - - # if not check_installation("cpupower",os_info): - env['MLC_INSTALL_CPUPOWER'] = 'True' + if os_info['platform'] == 'linux': + if not check_installation("numactl", os_info): + env['MLC_INSTALL_NUMACTL'] = 'True' + env['MLC_INSTALL_CPUPOWER'] = 'True' if env.get('MLC_PLATFORM_DETAILS_FILE_PATH', '') == '': if env.get('MLC_PLATFORM_DETAILS_DIR_PATH', '') == '': @@ -31,17 +32,961 @@ def preprocess(i): env['MLC_PLATFORM_DETAILS_FILE_PATH'] = os.path.join( env['MLC_PLATFORM_DETAILS_DIR_PATH'], env['MLC_PLATFORM_DETAILS_FILE_NAME']) + # Derive the raw file path from the parsed file path + base, ext = os.path.splitext(env['MLC_PLATFORM_DETAILS_FILE_PATH']) + env['MLC_PLATFORM_DETAILS_RAW_FILE_PATH'] = base + '-raw' + ext + return {'return': 0} def postprocess(i): state = i['state'] - env = i['env'] - os_info = i['os_info'] - automation = i['automation'] + raw_path = env.get('MLC_PLATFORM_DETAILS_RAW_FILE_PATH', '') + parsed_path = env.get('MLC_PLATFORM_DETAILS_FILE_PATH', '') + + if not raw_path or not os.path.isfile(raw_path): + return {'return': 0} + + try: + with open(raw_path, 'r') as f: + raw = json.load(f) + except Exception: + return {'return': 0} + + structured = _build_structured(raw, os_info.get('platform', 'linux')) + + with open(parsed_path, 'w') as f: + json.dump(structured, f, indent=2) + return {'return': 0} + + +# ===================================================================== +# Parsing helpers +# ===================================================================== + +def _safe_int(val, default=0): + try: + return int(val) + except (ValueError, TypeError): + return default + + +def _safe_float(val, default=0.0): + try: + return float(val) + except (ValueError, TypeError): + return default + + +def _get_output(raw, key): + """Get the output string for a raw entry, stripped.""" + entry = raw.get(key, {}) + if isinstance(entry, dict): + return entry.get('output', '').strip() + return '' + + +def _parse_key_value_colon(text): + """Parse lines of 'Key: Value' into a dict.""" + result = {} + for line in text.splitlines(): + if ':' in line: + key, _, val = line.partition(':') + key = key.strip() + val = val.strip() + if key: + result[key] = val + return result + + +# ===================================================================== +# Linux parsers +# ===================================================================== + +def _parse_lscpu(text): + """Parse lscpu output into a structured dict.""" + kv = _parse_key_value_colon(text) + + cpu = { + 'architecture': kv.get('Architecture', ''), + 'op_modes': kv.get('CPU op-mode(s)', ''), + 'address_sizes': kv.get('Address sizes', ''), + 'byte_order': kv.get('Byte Order', ''), + 'total_cpus': _safe_int(kv.get('CPU(s)', '')), + 'online_cpus_list': kv.get('On-line CPU(s) list', ''), + 'vendor_id': kv.get('Vendor ID', ''), + 'model_name': kv.get('Model name', ''), + 'cpu_family': _safe_int(kv.get('CPU family', '')), + 'model': _safe_int(kv.get('Model', '')), + 'stepping': _safe_int(kv.get('Stepping', '')), + 'threads_per_core': _safe_int(kv.get('Thread(s) per core', '')), + 'cores_per_socket': _safe_int(kv.get('Core(s) per socket', '')), + 'sockets': _safe_int(kv.get('Socket(s)', '')), + 'cpu_max_mhz': _safe_float(kv.get('CPU max MHz', '')), + 'cpu_min_mhz': _safe_float(kv.get('CPU min MHz', '')), + 'bogomips': _safe_float(kv.get('BogoMIPS', '')), + 'virtualization': kv.get('Virtualization', ''), + 'flags': kv.get('Flags', '').split(), + } + + # Cache info + caches = {} + for label in ['L1d cache', 'L1i cache', 'L2 cache', 'L3 cache']: + if label in kv: + caches[label.replace(' cache', '').replace( + ' ', '_').lower()] = kv[label] + cpu['caches'] = caches + + # NUMA + cpu['numa_nodes'] = _safe_int(kv.get('NUMA node(s)', '')) + for k, v in kv.items(): + if k.startswith('NUMA node') and 'CPU(s)' in k: + node_id = k.replace('NUMA node', '').replace(' CPU(s)', '').strip() + cpu.setdefault('numa_cpu_map', {})[f'node{node_id}'] = v + + # Vulnerabilities + vulns = {} + for k, v in kv.items(): + if k.startswith('Vulnerability '): + vuln_name = k.replace('Vulnerability ', '') + vulns[vuln_name] = v + cpu['vulnerabilities'] = vulns + + return cpu + + +def _parse_meminfo(text): + """Parse /proc/meminfo into a clean structured dict.""" + raw_entries = {} + for line in text.splitlines(): + m = re.match(r'(\S+):\s+(\d+)\s*(\S*)', line) + if m: + raw_entries[m.group(1)] = _safe_int(m.group(2)) + + def _kb_to_gb(key): + v = raw_entries.get(key, 0) + return round(v / 1048576, 2) if v else 0 + + return { + 'total_kb': raw_entries.get('MemTotal', 0), + 'total_gb': _kb_to_gb('MemTotal'), + 'free_kb': raw_entries.get('MemFree', 0), + 'free_gb': _kb_to_gb('MemFree'), + 'available_kb': raw_entries.get('MemAvailable', 0), + 'available_gb': _kb_to_gb('MemAvailable'), + 'buffers_kb': raw_entries.get('Buffers', 0), + 'cached_kb': raw_entries.get('Cached', 0), + 'swap_total_kb': raw_entries.get('SwapTotal', 0), + 'swap_total_gb': _kb_to_gb('SwapTotal'), + 'swap_free_kb': raw_entries.get('SwapFree', 0), + 'swap_free_gb': _kb_to_gb('SwapFree'), + 'huge_pages_total': raw_entries.get('HugePages_Total', 0), + 'huge_pages_free': raw_entries.get('HugePages_Free', 0), + 'huge_page_size_kb': raw_entries.get('Hugepagesize', 0), + 'dirty_kb': raw_entries.get('Dirty', 0), + 'shmem_kb': raw_entries.get('Shmem', 0), + } + + +def _parse_os_release(text): + """Parse /etc/os-release or macOS sw_vers style output into a dict.""" + result = {} + for line in text.splitlines(): + if '=' in line: + key, _, val = line.partition('=') + result[key.strip()] = val.strip().strip('"') + return result + + +def _parse_numa(text): + """Parse numactl --hardware output.""" + result = {} + m = re.search(r'available:\s+(\d+)\s+nodes', text) + if m: + result['available_nodes'] = _safe_int(m.group(1)) + + nodes = [] + for node_match in re.finditer(r'node\s+(\d+)\s+cpus:\s*(.+)', text): + node_id = _safe_int(node_match.group(1)) + cpus = node_match.group(2).strip() + size_match = re.search(rf'node\s+{node_id}\s+size:\s+(\d+)\s+MB', text) + free_match = re.search(rf'node\s+{node_id}\s+free:\s+(\d+)\s+MB', text) + nodes.append({ + 'node_id': node_id, + 'cpus': cpus, + 'size_mb': _safe_int(size_match.group(1)) if size_match else 0, + 'free_mb': _safe_int(free_match.group(1)) if free_match else 0, + }) + result['nodes'] = nodes + return result + + +def _parse_cpu_cache_table(text): + """Parse lscpu --cache table output.""" + lines = text.strip().splitlines() + if not lines: + return [] + headers = lines[0].split() + caches = [] + for line in lines[1:]: + parts = line.split() + if len(parts) >= len(headers): + entry = {} + for i, h in enumerate(headers): + entry[h.lower()] = parts[i] + caches.append(entry) + return caches + + +def _parse_disk_layout(text): + """Parse lsblk output into a list of disks.""" + disks = [] + for line in text.strip().splitlines(): + parts = line.split(None, 6) + if len(parts) >= 3: + disk = {'name': parts[0], 'type': parts[1], 'size': parts[2]} + if len(parts) >= 4: + disk['model'] = parts[3] + if len(parts) >= 5: + disk['transport'] = parts[4] + if len(parts) >= 6: + disk['rotational'] = parts[5] + if len(parts) >= 7: + disk['vendor'] = parts[6] + disks.append(disk) + return disks + + +def _parse_ulimit(text): + """Parse ulimit -a output.""" + result = {} + for line in text.splitlines(): + m = re.match(r'(.+?)\s+\(([^)]+)\)\s+(\S+)$', line.strip()) + if m: + name = m.group(1).strip() + flag = m.group(2).strip() + value = m.group(3).strip() + key = name.lower().replace(' ', '_').replace('-', '_') + result[key] = {'description': name, 'flag': flag, 'value': value} + return result + + +def _parse_cpu_frequency(text): + """Parse cpupower frequency-info output.""" + result = {} + m = re.search(r'driver:\s+(\S+)', text) + if m: + result['driver'] = m.group(1) + m = re.search(r'hardware limits:\s+(.+)', text) + if m: + result['hardware_limits'] = m.group(1).strip() + m = re.search(r'available cpufreq governors:\s+(.+)', text) + if m: + result['available_governors'] = m.group(1).strip().split() + m = re.search( + r'current CPU frequency:.*?(\d[\d.]+\s*\w+)\s*\(asserted', text) + if m: + result['current_frequency'] = m.group(1).strip() + + # Current policy + m = re.search( + r'current policy:.*?frequency should be within (.+?) and ([\d.]+ [A-Za-z]+)', + text, + re.DOTALL) + if m: + result['policy_min'] = m.group(1).strip() + result['policy_max'] = m.group(2).strip() + + # Boost info + boost = {} + m = re.search(r'Supported:\s+(yes|no)', text) + if m: + boost['supported'] = m.group(1) == 'yes' + m = re.search(r'Active:\s+(yes|no)', text) + if m: + boost['active'] = m.group(1) == 'yes' + m = re.search(r'Maximum Frequency:\s+([\d.]+ [A-Za-z]+)', text) + if m: + boost['max_frequency'] = m.group(1).strip() + m = re.search(r'Nominal.*?Frequency:\s+([\d.]+ [A-Za-z]+)', text) + if m: + boost['nominal_frequency'] = m.group(1).strip() + m = re.search(r'Lowest Non-linear.*?Frequency:\s+([\d.]+ [A-Za-z]+)', text) + if m: + boost['lowest_nonlinear_frequency'] = m.group(1).strip() + m = re.search(r'Lowest Frequency:\s+([\d.]+ [A-Za-z]+)', text) + if m: + boost['lowest_frequency'] = m.group(1).strip() + if boost: + result['boost'] = boost + + return result + + +def _parse_dmidecode_bios(text): + """Parse dmidecode -t bios for BIOS info.""" + result = {} + kv = _parse_key_value_colon(text) + for key in ['Vendor', 'Version', 'Release Date', 'Address', 'Runtime Size', 'ROM Size', + 'BIOS Revision']: + if key in kv: + result[key.lower().replace(' ', '_')] = kv[key] + return result + + +def _parse_dmidecode_memory(text): + """Parse dmidecode output for Memory Device entries (DMI type 17).""" + dimms = [] + sections = re.split(r'\nHandle ', text) + for section in sections: + if 'Memory Device' not in section: + continue + kv = _parse_key_value_colon(section) + size = kv.get('Size', '') + if not size or size == 'No Module Installed': + continue + if 'Type' not in kv: + continue + dimm = {} + for key in ['Size', 'Form Factor', 'Locator', 'Bank Locator', 'Type', + 'Speed', 'Manufacturer', 'Part Number', 'Rank', + 'Configured Memory Speed', 'Minimum Voltage', 'Maximum Voltage', + 'Configured Voltage', 'Memory Technology', 'Total Width', 'Data Width', + 'Serial Number']: + if key in kv: + dimm[key.lower().replace(' ', '_')] = kv[key] + dimms.append(dimm) + return dimms + + +def _parse_dmidecode_system(text): + """Parse dmidecode for System Information, Base Board, and Chassis.""" + result = {} + sections = re.split(r'\nHandle ', text) + for section in sections: + if 'System Information' in section: + kv = _parse_key_value_colon(section) + sys_info = {} + for key in ['Manufacturer', 'Product Name', 'Version', 'UUID', 'Wake-up Type', + 'SKU Number', 'Family']: + if key in kv: + val = kv[key] + if val != 'Default string': + sys_info[key.lower().replace( + ' ', '_').replace('-', '_')] = val + if sys_info: + result['system'] = sys_info + elif 'Base Board Information' in section: + kv = _parse_key_value_colon(section) + board = {} + for key in ['Manufacturer', 'Product Name', + 'Version', 'Serial Number', 'Type']: + if key in kv: + val = kv[key] + if val != 'Default string': + board[key.lower().replace(' ', '_')] = val + if board: + result['baseboard'] = board + elif 'Chassis Information' in section: + kv = _parse_key_value_colon(section) + chassis = {} + for key in ['Manufacturer', 'Type', 'Lock']: + if key in kv: + val = kv[key] + if val != 'Default string': + chassis[key.lower()] = val + if chassis: + result['chassis'] = chassis + return result + + +def _parse_thp(text): + """Parse transparent hugepage setting. E.g. '[always] madvise never' -> 'always'.""" + m = re.search(r'\[(\w+)\]', text) + if m: + return m.group(1) + return text.strip() + + +def _parse_cpu_vulnerabilities(text): + """Parse /sys/devices/system/cpu/vulnerabilities/ output.""" + result = {} + for line in text.splitlines(): + m = re.match(r'.*/vulnerabilities/(\S+):(.+)', line) + if m: + result[m.group(1).strip()] = m.group(2).strip() + return result + + +def _parse_memory_free_human(text): + """Parse 'free -h' output into structured dict.""" + result = {} + lines = text.strip().splitlines() + if len(lines) < 2: + return result + headers = lines[0].split() + for line in lines[1:]: + parts = line.split() + if not parts: + continue + row_name = parts[0].rstrip(':').lower() + row = {} + for i, h in enumerate(headers): + if i + 1 < len(parts): + row[h.lower()] = parts[i + 1] + result[row_name] = row + return result + + +def _parse_sysctl_key_settings(text): + """Extract commonly interesting sysctl settings from sysctl -a output.""" + interesting_keys = [ + 'vm.swappiness', 'vm.dirty_ratio', 'vm.dirty_background_ratio', + 'vm.overcommit_memory', 'vm.overcommit_ratio', 'vm.nr_hugepages', + 'vm.max_map_count', 'vm.zone_reclaim_mode', + 'kernel.shmmax', 'kernel.shmall', 'kernel.shmmni', + 'kernel.threads-max', 'kernel.pid_max', 'kernel.sched_migration_cost_ns', + 'kernel.numa_balancing', 'kernel.randomize_va_space', + 'net.core.somaxconn', 'net.core.rmem_max', 'net.core.wmem_max', + 'net.ipv4.tcp_max_syn_backlog', 'net.ipv4.ip_local_port_range', + ] + kv = {} + for line in text.splitlines(): + if ' = ' in line: + key, _, val = line.partition(' = ') + kv[key.strip()] = val.strip() + return {k: kv[k] for k in interesting_keys if k in kv} + + +# ===================================================================== +# macOS parsers +# ===================================================================== + +def _parse_sysctl_hw(text): + """Parse macOS 'sysctl hw' output into a CPU/hardware dict.""" + kv = {} + for line in text.splitlines(): + if ':' in line: + key, _, val = line.partition(':') + kv[key.strip()] = val.strip() + + cpu = { + 'model_name': kv.get('machdep.cpu.brand_string', ''), + 'total_cpus': _safe_int(kv.get('hw.ncpu', kv.get('hw.logicalcpu', ''))), + 'physical_cpus': _safe_int(kv.get('hw.physicalcpu', '')), + 'logical_cpus': _safe_int(kv.get('hw.logicalcpu', '')), + 'cpu_family': kv.get('machdep.cpu.family', ''), + 'cpu_model': kv.get('machdep.cpu.model', ''), + 'stepping': kv.get('machdep.cpu.stepping', ''), + 'vendor_id': kv.get('machdep.cpu.vendor', ''), + 'cpu_freq_hz': _safe_int(kv.get('hw.cpufrequency', '')), + 'l1d_cache': _safe_int(kv.get('hw.l1dcachesize', '')), + 'l1i_cache': _safe_int(kv.get('hw.l1icachesize', '')), + 'l2_cache': _safe_int(kv.get('hw.l2cachesize', '')), + 'l3_cache': _safe_int(kv.get('hw.l3cachesize', '')), + 'memory_bytes': _safe_int(kv.get('hw.memsize', '')), + } + + mem_bytes = cpu.pop('memory_bytes', 0) + memory = {} + if mem_bytes: + memory['total_bytes'] = mem_bytes + memory['total_gb'] = round(mem_bytes / (1024 ** 3), 2) + + return cpu, memory + + +def _parse_system_profiler(text): + """Parse macOS system_profiler SPHardwareDataType output.""" + kv = _parse_key_value_colon(text) + result = {} + for key in ['Model Name', 'Model Identifier', 'Chip', 'Total Number of Cores', + 'Memory', 'System Firmware Version', 'OS Loader Version', + 'Serial Number (system)', 'Hardware UUID', 'Provisioning UDID']: + if key in kv: + result[key.lower().replace(' ', '_').replace( + '(', '').replace(')', '')] = kv[key] + return result + + +def _parse_sw_vers(text): + """Parse macOS sw_vers output.""" + result = {} + for line in text.splitlines(): + if ':' in line: + key, _, val = line.partition(':') + result[key.strip().lower().replace(' ', '_')] = val.strip() + return result + + +def _parse_diskutil(text): + """Parse macOS diskutil list output into a simple list.""" + disks = [] + current_disk = None + for line in text.splitlines(): + m = re.match(r'^(/dev/\S+)\s+\((\w+).*?\):', line) + if m: + current_disk = {'device': m.group(1), 'type': m.group(2)} + disks.append(current_disk) + elif current_disk and line.strip(): + m2 = re.match( + r'\s+\d+:\s+(\S+)\s+(.+?)\s+([\d.]+\s+\S+)\s+(\S+)', line) + if m2: + current_disk.setdefault('partitions', []).append({ + 'type': m2.group(1), + 'name': m2.group(2).strip(), + 'size': m2.group(3), + 'identifier': m2.group(4), + }) + return disks + + +def _parse_macos_sysctl_settings(text): + """Extract interesting macOS sysctl settings.""" + interesting_keys = [ + 'kern.maxproc', 'kern.maxfiles', 'kern.maxfilesperproc', + 'kern.hostname', 'kern.ostype', 'kern.osrelease', 'kern.osrevision', + 'kern.version', 'kern.boottime', 'vm.swapusage', + 'machdep.cpu.core_count', 'machdep.cpu.thread_count', + 'machdep.cpu.features', 'machdep.cpu.leaf7_features', + ] + kv = {} + for line in text.splitlines(): + if ':' in line: + key, _, val = line.partition(':') + kv[key.strip()] = val.strip() + return {k: kv[k] for k in interesting_keys if k in kv} + + +# ===================================================================== +# Windows parsers +# ===================================================================== + +def _parse_systeminfo(text): + """Parse Windows systeminfo output.""" + kv = _parse_key_value_colon(text) + result = {} + for key in ['Host Name', 'OS Name', 'OS Version', 'OS Manufacturer', + 'OS Configuration', 'OS Build Type', 'Registered Owner', + 'System Manufacturer', 'System Model', 'System Type', + 'BIOS Version', 'Total Physical Memory', 'Available Physical Memory', + 'Virtual Memory: Max Size', 'Virtual Memory: Available', + 'Virtual Memory: In Use', 'Domain', 'Logon Server', + 'Boot Device', 'System Directory', 'Original Install Date', + 'System Boot Time', 'Time Zone']: + if key in kv: + result[key.lower().replace(' ', '_').replace( + ':', '_').replace('__', '_')] = kv[key] + return result + + +def _parse_windows_cpu(text): + """Parse Windows CPU info (from Get-CimInstance or wmic).""" + result = {} + kv = _parse_key_value_colon(text) + for key in ['Name', 'Manufacturer', 'MaxClockSpeed', 'CurrentClockSpeed', + 'NumberOfCores', 'NumberOfLogicalProcessors', 'L2CacheSize', + 'L3CacheSize', 'Architecture', 'Caption', 'DeviceID', + 'ProcessorId', 'SocketDesignation', 'AddressWidth', + 'VirtualizationFirmwareEnabled']: + if key in kv: + result[key.lower()] = kv[key] + # Unify + cpu = { + 'model_name': result.get('name', result.get('caption', '')), + 'manufacturer': result.get('manufacturer', ''), + 'total_cpus': _safe_int(result.get('numberoflogicalprocessors', '')), + 'cores': _safe_int(result.get('numberofcores', '')), + 'max_clock_mhz': _safe_int(result.get('maxclockspeed', '')), + 'current_clock_mhz': _safe_int(result.get('currentclockspeed', '')), + 'l2_cache_kb': _safe_int(result.get('l2cachesize', '')), + 'l3_cache_kb': _safe_int(result.get('l3cachesize', '')), + 'socket': result.get('socketdesignation', ''), + 'address_width': _safe_int(result.get('addresswidth', '')), + } + return cpu + + +def _parse_windows_memory(text): + """Parse Windows memory info (from Get-CimInstance Win32_PhysicalMemory).""" + dimms = [] + current = {} + for line in text.splitlines(): + line = line.strip() + if not line: + if current: + dimms.append(current) + current = {} + continue + if ':' in line: + key, _, val = line.partition(':') + key = key.strip().lower() + val = val.strip() + if key and val: + current[key] = val + if current: + dimms.append(current) + + result = [] + for d in dimms: + dimm = {} + capacity = _safe_int(d.get('capacity', '0')) + if capacity: + dimm['size_gb'] = round(capacity / (1024 ** 3), 2) + for k in ['manufacturer', 'partnumber', 'speed', 'memorytype', + 'formfactor', 'banklabel', 'devicelocator', 'serialnumber']: + if k in d: + dimm[k] = d[k] + if dimm: + result.append(dimm) + return result + + +def _parse_windows_disk(text): + """Parse Windows disk info (from Get-CimInstance Win32_DiskDrive).""" + disks = [] + current = {} + for line in text.splitlines(): + line = line.strip() + if not line: + if current: + disks.append(current) + current = {} + continue + if ':' in line: + key, _, val = line.partition(':') + key = key.strip().lower() + val = val.strip() + if key and val: + current[key] = val + if current: + disks.append(current) + + result = [] + for d in disks: + disk = {} + size = _safe_int(d.get('size', '0')) + if size: + disk['size_gb'] = round(size / (1024 ** 3), 2) + for k in ['model', 'mediatype', + 'interfacetype', 'serialnumber', 'caption']: + if k in d: + disk[k] = d[k] + if disk: + result.append(disk) + return result + + +# ===================================================================== +# Build structured JSON +# ===================================================================== + +def _build_structured(raw, platform='linux'): + """Build a well-structured JSON from the raw command outputs.""" + structured = {'platform': platform} + + if platform == 'linux': + _build_linux(raw, structured) + elif platform == 'darwin': + _build_macos(raw, structured) + elif platform == 'windows': + _build_windows(raw, structured) + + return structured + + +def _build_linux(raw, structured): + """Parse Linux command outputs.""" + # CPU + lscpu_text = _get_output(raw, 'lscpu') + if lscpu_text: + structured['cpu'] = _parse_lscpu(lscpu_text) + + cache_text = _get_output(raw, 'cpu_cache') + if cache_text: + structured['cpu_cache_topology'] = _parse_cpu_cache_table(cache_text) + + freq_text = _get_output(raw, 'cpu_frequency') + if freq_text: + structured['cpu_frequency'] = _parse_cpu_frequency(freq_text) + + vuln_text = _get_output(raw, 'cpu_vulnerabilities') + if vuln_text: + structured['cpu_vulnerabilities'] = _parse_cpu_vulnerabilities( + vuln_text) + + # Memory + meminfo_text = _get_output(raw, 'memory_info') + if meminfo_text: + structured['memory'] = _parse_meminfo(meminfo_text) + + memfree_text = _get_output(raw, 'memory_free_human') + if memfree_text: + structured['memory_human'] = _parse_memory_free_human(memfree_text) + + # DMI + dmi_text = _get_output(raw, 'dmidecode_full') + if dmi_text: + dimms = _parse_dmidecode_memory(dmi_text) + if dimms: + structured['memory_dimms'] = dimms + hw_info = _parse_dmidecode_system(dmi_text) + if hw_info: + structured['hardware'] = hw_info + + bios_text = _get_output(raw, 'bios_info') + if bios_text: + structured['bios'] = _parse_dmidecode_bios(bios_text) + + # OS + os_release_text = _get_output(raw, 'os_release') + if os_release_text: + structured['os'] = _parse_os_release(os_release_text) + + op_sys = _get_output(raw, 'operating_system') + if op_sys: + structured.setdefault('os', {})['mlc_os_string'] = op_sys + + lsb = _get_output(raw, 'lsb_release') + if lsb: + structured.setdefault('os', {})['lsb_description'] = lsb + + # Kernel + kernel = {} + uname = _get_output(raw, 'kernel_and_arch') + if uname: + kernel['uname'] = uname + parts = uname.split() + if len(parts) >= 3: + kernel['version'] = parts[2] + cmdline = _get_output(raw, 'kernel_cmdline') + if cmdline: + kernel['cmdline'] = cmdline + if kernel: + structured['kernel'] = kernel + + # NUMA + numa_text = _get_output(raw, 'numa_topology') + if numa_text: + structured['numa'] = _parse_numa(numa_text) + + # THP + thp = {} + thp_enabled = _get_output(raw, 'thp_enabled') + if thp_enabled: + thp['enabled'] = _parse_thp(thp_enabled) + thp_defrag = _get_output(raw, 'thp_defrag') + if thp_defrag: + thp['defrag'] = thp_defrag + if thp: + structured['transparent_hugepages'] = thp + + # Disks + disk_text = _get_output(raw, 'disk_layout') + if disk_text: + structured['disks'] = _parse_disk_layout(disk_text) + + # ulimit + ulimit_text = _get_output(raw, 'ulimit_settings') + if ulimit_text: + structured['ulimits'] = _parse_ulimit(ulimit_text) + + # Systemd + systemd = {} + ver = _get_output(raw, 'systemd_version') + if ver: + systemd['version'] = ver + units_text = _get_output(raw, 'systemd_units') + if units_text: + enabled = len(re.findall(r'\s+enabled\s', units_text)) + disabled = len(re.findall(r'\s+disabled\s', units_text)) + static = len(re.findall(r'\s+static\s', units_text)) + systemd['unit_counts'] = { + 'enabled': enabled, + 'disabled': disabled, + 'static': static} + if systemd: + structured['systemd'] = systemd + + # Sysctl + sysctl_text = _get_output(raw, 'sysctl_all') + if sysctl_text: + sysctl = _parse_sysctl_key_settings(sysctl_text) + if sysctl: + structured['sysctl'] = sysctl + + # User / uptime + user = _get_output(raw, 'current_user') + if user: + structured['current_user'] = user + + w_text = _get_output(raw, 'logged_in_users') + if w_text: + m = re.search(r'up\s+(.+?),\s+\d+\s+user', w_text) + if m: + structured['uptime'] = m.group(1).strip() + m = re.search(r'load average:\s+(.+)', w_text) + if m: + structured['load_average'] = m.group(1).strip() + + # Runlevel + runlevel_text = _get_output(raw, 'runlevel') + if runlevel_text: + m = re.search(r'run-level\s+(\d+)', runlevel_text) + if m: + structured['runlevel'] = _safe_int(m.group(1)) + + # PCI devices + pci_text = _get_output(raw, 'pci_devices') + if pci_text and 'FAILED' not in pci_text: + devices = [] + for line in pci_text.splitlines(): + m = re.match(r'(\S+)\s+(.+?):\s+(.+)', line) + if m: + devices.append({ + 'slot': m.group(1), + 'class': m.group(2).strip(), + 'device': m.group(3).strip(), + }) + if devices: + structured['pci_devices'] = devices + + # Infiniband + ib_text = _get_output(raw, 'infiniband_status') + if ib_text and 'FAILED' not in ib_text and 'not found' not in ib_text.lower(): + structured['infiniband'] = ib_text + + # GPU (CUDA) + _parse_gpu_entries(raw, structured) + + +def _build_macos(raw, structured): + """Parse macOS command outputs.""" + # CPU from sysctl hw + hw_text = _get_output(raw, 'sysctl_hw') + if hw_text: + cpu, memory = _parse_sysctl_hw(hw_text) + structured['cpu'] = cpu + if memory: + structured['memory'] = memory + + # System profiler + profiler_text = _get_output(raw, 'system_profiler') + if profiler_text: + structured['hardware'] = _parse_system_profiler(profiler_text) + + # sw_vers + swvers_text = _get_output(raw, 'sw_vers') + if swvers_text: + structured['os'] = _parse_sw_vers(swvers_text) + + op_sys = _get_output(raw, 'operating_system') + if op_sys: + structured.setdefault('os', {})['mlc_os_string'] = op_sys + + # Kernel + uname = _get_output(raw, 'kernel_and_arch') + if uname: + parts = uname.split() + structured['kernel'] = { + 'uname': uname, + 'version': parts[2] if len(parts) >= 3 else '', + } + + # Disks + disk_text = _get_output(raw, 'diskutil') + if disk_text: + structured['disks'] = _parse_diskutil(disk_text) + + # ulimit + ulimit_text = _get_output(raw, 'ulimit_settings') + if ulimit_text: + structured['ulimits'] = _parse_ulimit(ulimit_text) + + # Sysctl settings + sysctl_text = _get_output(raw, 'sysctl_all') + if sysctl_text: + sysctl = _parse_macos_sysctl_settings(sysctl_text) + if sysctl: + structured['sysctl'] = sysctl + + # User / uptime + user = _get_output(raw, 'current_user') + if user: + structured['current_user'] = user + + w_text = _get_output(raw, 'logged_in_users') + if w_text: + m = re.search(r'up\s+(.+?),\s+\d+\s+user', w_text) + if m: + structured['uptime'] = m.group(1).strip() + m = re.search(r'load average[s]?:\s+(.+)', w_text) + if m: + structured['load_average'] = m.group(1).strip() + + # GPU + _parse_gpu_entries(raw, structured) + + +def _build_windows(raw, structured): + """Parse Windows command outputs.""" + # systeminfo + sysinfo_text = _get_output(raw, 'systeminfo') + if sysinfo_text: + structured['system'] = _parse_systeminfo(sysinfo_text) + + # CPU + cpu_text = _get_output(raw, 'cpu_info') + if cpu_text: + structured['cpu'] = _parse_windows_cpu(cpu_text) + + # OS + os_text = _get_output(raw, 'os_info') + if os_text: + structured['os'] = _parse_key_value_colon(os_text) + + op_sys = _get_output(raw, 'operating_system') + if op_sys: + structured.setdefault('os', {})['mlc_os_string'] = op_sys + + # Memory + mem_text = _get_output(raw, 'memory_info') + if mem_text: + structured['memory_dimms'] = _parse_windows_memory(mem_text) + + # Disks + disk_text = _get_output(raw, 'disk_info') + if disk_text: + structured['disks'] = _parse_windows_disk(disk_text) + + # Kernel + uname = _get_output(raw, 'kernel_and_arch') + if uname: + structured['kernel'] = {'version': uname} + + # User + user = _get_output(raw, 'current_user') + if user: + structured['current_user'] = user + + # GPU + _parse_gpu_entries(raw, structured) + + +def _parse_gpu_entries(raw, structured): + """Parse GPU/accelerator entries (shared across platforms).""" + gpu_topo = _get_output(raw, 'nvidia_smi_topo') + gpu_full = _get_output(raw, 'nvidia_smi_full') + if gpu_topo or gpu_full: + gpu = {} + if gpu_topo: + gpu['topology'] = gpu_topo + if gpu_full: + gpu['nvidia_smi_query'] = gpu_full + structured['gpu'] = gpu diff --git a/script/get-platform-details/meta.yaml b/script/get-platform-details/meta.yaml index 82070b1a0..9db809c34 100644 --- a/script/get-platform-details/meta.yaml +++ b/script/get-platform-details/meta.yaml @@ -8,6 +8,7 @@ deps: - skip_if_env: MLC_HOST_OS_TYPE: - windows + - darwin tags: detect,sudo - skip_if_any_env: MLC_HOST_OS_TYPE: @@ -19,6 +20,7 @@ deps: - skip_if_any_env: MLC_HOST_OS_TYPE: - windows + - darwin tags: get,sys-util,generic,_jq - enable_if_env: MLC_HOST_OS_TYPE: @@ -39,6 +41,7 @@ input_mapping: out_file_name: MLC_PLATFORM_DETAILS_FILE_NAME new_env_keys: - MLC_PLATFORM_DETAILS_FILE_PATH + - MLC_PLATFORM_DETAILS_RAW_FILE_PATH prehook_deps: - enable_if_env: MLC_HOST_OS_TYPE: @@ -67,6 +70,9 @@ tags: - details - platform-details uid: f0801943c17f4e48 +tests: + run_inputs: + - quiet: true variations: cuda: group: gpu-driver diff --git a/script/get-platform-details/run-macos.sh b/script/get-platform-details/run-macos.sh index fcde181c0..0c3e701df 100644 --- a/script/get-platform-details/run-macos.sh +++ b/script/get-platform-details/run-macos.sh @@ -1 +1,96 @@ -echo "WARNING: get-platform-details script is fully supported on linux systems only." +#!/bin/bash + +OUTPUT_FILE="$MLC_PLATFORM_DETAILS_RAW_FILE_PATH" + +# Start with empty JSON object +echo '{}' > "$OUTPUT_FILE" + +add_entry () { + local key="$1" + local cmd="$2" + local needs_sudo="$3" + + local tmpfile + tmpfile=$(mktemp) + + if [[ "$needs_sudo" == "yes" ]]; then + if [[ ${MLC_SUDO_USER} == "yes" ]]; then + ${MLC_SUDO} bash -c "$cmd" > "$tmpfile" 2>&1 || echo "FAILED (sudo): $cmd" > "$tmpfile" + else + echo "sudo not available" > "$tmpfile" + fi + else + bash -c "$cmd" > "$tmpfile" 2>&1 || echo "FAILED: $cmd" > "$tmpfile" + fi + + # macOS ships with an older jq or none; use python as fallback + if command -v jq >/dev/null 2>&1; then + jq --arg k "$key" \ + --arg c "$cmd" \ + --rawfile o "$tmpfile" \ + '. + {($k): {"command": $c, "output": $o}}' \ + "$OUTPUT_FILE" > "$OUTPUT_FILE.tmp" && mv "$OUTPUT_FILE.tmp" "$OUTPUT_FILE" + else + python3 -c " +import json, sys +with open('$OUTPUT_FILE') as f: + data = json.load(f) +with open('$tmpfile') as f: + output = f.read() +data['$key'] = {'command': '''$cmd''', 'output': output} +with open('$OUTPUT_FILE', 'w') as f: + json.dump(data, f, indent=2) +" + fi + + rm -f "$tmpfile" +} + +# -------- Kernel / OS -------- +add_entry "kernel_and_arch" "uname -a" no +add_entry "sw_vers" "sw_vers" no +add_entry "current_user" "whoami" no +add_entry "logged_in_users" "w" no +add_entry "operating_system" "echo $MLC_HOST_OS_TYPE-$MLC_HOST_OS_FLAVOR_LIKE-$MLC_HOST_OS_FLAVOR-$MLC_HOST_OS_VERSION" no + +# -------- CPU / Hardware -------- +add_entry "sysctl_hw" "sysctl hw machdep.cpu" no +add_entry "system_profiler" "system_profiler SPHardwareDataType" no + +# -------- Memory -------- +add_entry "vm_stat" "vm_stat" no + +# -------- Disk -------- +add_entry "diskutil" "diskutil list" no +add_entry "df_h" "df -h" no + +# -------- Network -------- +add_entry "ifconfig" "ifconfig -a" no + +# -------- Limits -------- +add_entry "ulimit_settings" "ulimit -a" no +add_entry "launchctl_limit" "launchctl limit" no + +# -------- Sysctl (full) -------- +add_entry "sysctl_all" "sysctl -a" no + +# -------- Power -------- +add_entry "power_settings" "pmset -g" no + +# -------- GPU -------- +add_entry "gpu_info" "system_profiler SPDisplaysDataType" no + +# ------------------------------------------------------------------- +# Accelerator detection (CUDA only) +# ------------------------------------------------------------------- +if [[ "$MLC_ACCELERATOR_BACKEND" == "cuda" ]]; then + if command -v nvidia-smi >/dev/null 2>&1; then + add_entry "nvidia_smi_topo" "nvidia-smi topo -m" no + add_entry "nvidia_smi_full" "nvidia-smi -q" no + else + add_entry "accelerator_error" \ + "echo nvidia-smi not found; cannot detect CUDA accelerator details" no + fi +fi + +echo "Raw system information written to $OUTPUT_FILE" diff --git a/script/get-platform-details/run.bat b/script/get-platform-details/run.bat index fcde181c0..a0d2ac77e 100644 --- a/script/get-platform-details/run.bat +++ b/script/get-platform-details/run.bat @@ -1 +1,82 @@ -echo "WARNING: get-platform-details script is fully supported on linux systems only." +@echo off +setlocal enabledelayedexpansion + +set "OUTPUT_FILE=%MLC_PLATFORM_DETAILS_RAW_FILE_PATH%" + +:: Initialize empty JSON via Python +python -c "import json; json.dump({}, open(r'%OUTPUT_FILE%', 'w'), indent=2)" + +:: Helper: run a command and add its output to the JSON file using Python +:: Usage: call :add_entry KEY "COMMAND" +:: We use Python to safely handle JSON escaping + +call :add_entry "kernel_and_arch" "ver" +call :add_entry "current_user" "whoami" +call :add_entry "operating_system" "echo %MLC_HOST_OS_TYPE%-%MLC_HOST_OS_FLAVOR%-%MLC_HOST_OS_VERSION%" + +:: CPU info via PowerShell +call :add_entry "cpu_info" "powershell -NoProfile -Command ""Get-CimInstance Win32_Processor | Format-List Name,Manufacturer,MaxClockSpeed,CurrentClockSpeed,NumberOfCores,NumberOfLogicalProcessors,L2CacheSize,L3CacheSize,Architecture,Caption,DeviceID,SocketDesignation,AddressWidth,VirtualizationFirmwareEnabled | Out-String""" + +:: OS info via PowerShell +call :add_entry "os_info" "powershell -NoProfile -Command ""Get-CimInstance Win32_OperatingSystem | Format-List Caption,Version,BuildNumber,OSArchitecture,TotalVisibleMemorySize,FreePhysicalMemory,InstallDate,LastBootUpTime,SystemDirectory | Out-String""" + +:: systeminfo +call :add_entry "systeminfo" "systeminfo" + +:: Memory DIMMs via PowerShell +call :add_entry "memory_info" "powershell -NoProfile -Command ""Get-CimInstance Win32_PhysicalMemory | Format-List Capacity,Manufacturer,PartNumber,Speed,MemoryType,FormFactor,BankLabel,DeviceLocator,SerialNumber | Out-String""" + +:: Disk info via PowerShell +call :add_entry "disk_info" "powershell -NoProfile -Command ""Get-CimInstance Win32_DiskDrive | Format-List Model,Size,MediaType,InterfaceType,SerialNumber,Caption | Out-String""" + +:: Network adapters +call :add_entry "network_adapters" "powershell -NoProfile -Command ""Get-NetAdapter | Format-List Name,InterfaceDescription,Status,LinkSpeed,MacAddress | Out-String""" + +:: Environment variables (selected) +call :add_entry "env_info" "powershell -NoProfile -Command ""Get-ChildItem Env: | Where-Object { $_.Name -match 'PROCESSOR|NUMBER_OF_PROCESSORS|OS|COMPUTERNAME|USERNAME|USERDOMAIN' } | Format-List | Out-String""" + +:: BIOS info via PowerShell +call :add_entry "bios_info" "powershell -NoProfile -Command ""Get-CimInstance Win32_BIOS | Format-List Manufacturer,Name,Version,ReleaseDate,SMBIOSBIOSVersion | Out-String""" + +:: Baseboard info +call :add_entry "baseboard_info" "powershell -NoProfile -Command ""Get-CimInstance Win32_BaseBoard | Format-List Manufacturer,Product,Version,SerialNumber | Out-String""" + +:: GPU info via PowerShell +call :add_entry "gpu_info" "powershell -NoProfile -Command ""Get-CimInstance Win32_VideoController | Format-List Name,AdapterRAM,DriverVersion,VideoProcessor,CurrentHorizontalResolution,CurrentVerticalResolution | Out-String""" + +:: ------------------------------------------------------------------- +:: Accelerator detection (CUDA only) +:: ------------------------------------------------------------------- +if "%MLC_ACCELERATOR_BACKEND%"=="cuda" ( + where nvidia-smi >nul 2>&1 + if !errorlevel! equ 0 ( + call :add_entry "nvidia_smi_topo" "nvidia-smi topo -m" + call :add_entry "nvidia_smi_full" "nvidia-smi -q" + ) else ( + call :add_entry "accelerator_error" "echo nvidia-smi not found" + ) +) + +echo Raw system information written to %OUTPUT_FILE% +exit /b 0 + +:add_entry +:: %~1 = key, %~2 = command +set "ENTRY_KEY=%~1" +set "ENTRY_CMD=%~2" + +:: Create temp files for command output and command itself +set "TMPFILE=%TEMP%\mlc_platform_%RANDOM%.txt" +set "TMPCMD=%TEMP%\mlc_cmd_%RANDOM%.txt" + +:: Execute command directly using delayed expansion to handle pipes correctly +!ENTRY_CMD! > "%TMPFILE%" 2>&1 + +:: Write command to temp file to avoid escaping issues in Python +echo !ENTRY_CMD!> "%TMPCMD%" + +:: Use Python to safely add the entry to JSON (reads command from file to handle all special chars) +python -c "import json; f=open(r'%OUTPUT_FILE%'); data=json.load(f); f.close(); g=open(r'%TMPFILE%', encoding='utf-8', errors='replace'); out=g.read(); g.close(); h=open(r'%TMPCMD%', encoding='utf-8', errors='replace'); cmd=h.read().strip(); h.close(); data[r'%ENTRY_KEY%']={'command': cmd, 'output': out}; j=open(r'%OUTPUT_FILE%','w'); json.dump(data, j, indent=2); j.close()" + +del /f /q "%TMPFILE%" "%TMPCMD%" >nul 2>&1 +exit /b 0 diff --git a/script/get-platform-details/run.sh b/script/get-platform-details/run.sh index 7eabee2bf..9e1285cd2 100644 --- a/script/get-platform-details/run.sh +++ b/script/get-platform-details/run.sh @@ -1,18 +1,8 @@ #!/bin/bash -OUTPUT_FILE="$MLC_PLATFORM_DETAILS_FILE_PATH" +OUTPUT_FILE="$MLC_PLATFORM_DETAILS_RAW_FILE_PATH" -# Start with empty JSON object. We are using jq to populate the JSON file with a structure like: -# { -# "key1": { -# "command": "the command we ran", -# "output": "the output of the command" -# }, -# "key2": { -# "command": "another command", -# "output": "its output" -# } -# } +# Start with empty JSON object echo '{}' > "$OUTPUT_FILE" add_entry () { @@ -20,7 +10,6 @@ add_entry () { local cmd="$2" local needs_sudo="$3" - local output local tmpfile tmpfile=$(mktemp) @@ -39,7 +28,7 @@ add_entry () { --rawfile o "$tmpfile" \ '. + {($k): {"command": $c, "output": $o}}' \ "$OUTPUT_FILE" > "$OUTPUT_FILE.tmp" && mv "$OUTPUT_FILE.tmp" "$OUTPUT_FILE" - + rm -f "$tmpfile" } @@ -69,7 +58,7 @@ add_entry "numa_topology" "numactl --hardware" no add_entry "cpu_frequency" "cpupower frequency-info" no add_entry "memory_free_human" "free -h" no add_entry "pci_devices" "lspci" no -add_entry "infiniband_status" "ibstat" no # requires Infiniband drivers; will fail gracefully if not present +add_entry "infiniband_status" "ibstat" no add_entry "operating_system" "echo $MLC_HOST_OS_TYPE-$MLC_HOST_OS_FLAVOR_LIKE-$MLC_HOST_OS_FLAVOR-$MLC_HOST_OS_VERSION" no # -------- Sudo-required commands -------- @@ -82,19 +71,13 @@ add_entry "bios_info" "dmidecode -t bios" yes # Accelerator detection (CUDA only) # ------------------------------------------------------------------- if [[ "$MLC_ACCELERATOR_BACKEND" == "cuda" ]]; then - if command -v nvidia-smi >/dev/null 2>&1; then - - add_entry "nvidia_smi_topo" \ - "nvidia-smi topo -m" no - - add_entry "nvidia_smi_full" \ - "nvidia-smi -q" no - + add_entry "nvidia_smi_topo" "nvidia-smi topo -m" no + add_entry "nvidia_smi_full" "nvidia-smi -q" no else add_entry "accelerator_error" \ "echo nvidia-smi not found; cannot detect CUDA accelerator details" no fi fi -echo "System information written to $OUTPUT_FILE" +echo "Raw system information written to $OUTPUT_FILE" From 758e0461aa2aba8b8feca9ddbec5c00b4849f7f3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 19 Mar 2026 19:36:05 +0000 Subject: [PATCH 46/53] [Automated Commit] Document script/benchmark-program-geekbench/meta.yaml [skip ci] --- script/benchmark-program-geekbench/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/script/benchmark-program-geekbench/README.md b/script/benchmark-program-geekbench/README.md index 603087e81..9f37b9f95 100644 --- a/script/benchmark-program-geekbench/README.md +++ b/script/benchmark-program-geekbench/README.md @@ -43,7 +43,7 @@ mlcr benchmark,geekbench,benchmark-geekbench,benchmark-program-geekbench | `--license_email` | | | `` | | `--license_key` | | | `` | | `--workload` | | | `` | -| `--iterations` | | | `3` | +| `--iterations` | | | `` | | `--num_runs` | | | `1` | | `--core_pinning` | | | `no` | | `--pinned_core` | | | `0` | @@ -60,6 +60,8 @@ mlcr benchmark,geekbench,benchmark-geekbench,benchmark-program-geekbench | `--workload_ids` | | | `` | | `--cpu_workers` | | | `` | | `--workload_gap` | | | `` | +| `--reuse_logs` | | | `` | +| `--skip_platform_details` | | | `` | ### Generic Script Inputs | Name | Description | Choices | Default | From 7e0d35ef5119ee51798b8b7301ced718b8b8865c Mon Sep 17 00:00:00 2001 From: amd-arsuresh Date: Fri, 20 Mar 2026 15:36:51 +0000 Subject: [PATCH 47/53] Improvements to GB script (#869) --- script/README.md | 2 +- script/get-geekbench/customize.py | 24 +++++++++++++++--------- script/get-geekbench/meta.yaml | 12 +++++++++++- script/get-geekbench/run.sh | 1 - script/get-generic-sys-util/meta.yaml | 14 ++++++++++++++ 5 files changed, 41 insertions(+), 12 deletions(-) diff --git a/script/README.md b/script/README.md index 91780a074..131b03259 100644 --- a/script/README.md +++ b/script/README.md @@ -1,6 +1,6 @@ # MLCommons Automation Scripts -*Last updated: 2026-03-19 20:21:31* +*Last updated: 2026-03-20 17:33:41* This directory contains automation scripts for MLPerf benchmarks, AI/ML workflows, and development operations. diff --git a/script/get-geekbench/customize.py b/script/get-geekbench/customize.py index 2b5682df9..3660a67e1 100644 --- a/script/get-geekbench/customize.py +++ b/script/get-geekbench/customize.py @@ -84,13 +84,14 @@ def postprocess(i): # Windows: run.bat ran the installer and exported paths via tmp-run-env.out # MLC_GEEKBENCH_BIN_WITH_PATH is set from tmp-run-env.out geekbench_bin = env.get('MLC_GEEKBENCH_BIN_WITH_PATH', '') - install_dir = env.get('MLC_GEEKBENCH_WINDOWS_INSTALL_DIR', '') + install_dir = env.get('MLC_GEEKBENCH_INSTALLED_PATH', '') if geekbench_bin == '' or not os.path.isfile(geekbench_bin): # Fallback: try to find in the install dir if install_dir and os.path.isdir(install_dir): for f in os.listdir(install_dir): - if f.lower().startswith('geekbench6') and f.lower().endswith('.exe'): + if f.lower().startswith( + f'geekbench{env["MLC_GEEKBENCH_VERSION_MAJOR"]}') and f.lower().endswith('.exe'): geekbench_bin = os.path.join(install_dir, f) break @@ -103,16 +104,18 @@ def postprocess(i): else: # Linux / macOS: extracted archive - extracted_path = env.get('MLC_GEEKBENCH_EXTRACTED_PATH', '') - if extracted_path == '': + geekbench_dir = env.get( + 'MLC_GEEKBENCH_INSTALLED_PATH', env.get( + 'MLC_GEEKBENCH_EXTRACTED_PATH', '')) + if geekbench_dir == '' and env.get( + 'MLC_GEEKBENCH_INSTALLED_PATH', '') == '': return {'return': 1, 'error': 'MLC_GEEKBENCH_EXTRACTED_PATH not set - download-and-extract may have failed'} # Find the Geekbench directory inside the extracted path - geekbench_dir = extracted_path - if os.path.isdir(extracted_path): - for d in sorted(os.listdir(extracted_path)): - full = os.path.join(extracted_path, d) + if os.path.isdir(geekbench_dir): + for d in sorted(os.listdir(geekbench_dir)): + full = os.path.join(geekbench_dir, d) if os.path.isdir(full) and d.lower().startswith('geekbench'): geekbench_dir = full break @@ -120,7 +123,7 @@ def postprocess(i): if os_info['platform'] == 'darwin': # macOS: binary is inside the .app bundle # e.g. Geekbench 6.app/Contents/MacOS/Geekbench 6 - app_pattern = os.path.join(extracted_path, 'Geekbench*.app') + app_pattern = os.path.join(geekbench_dir, 'Geekbench*.app') app_matches = glob.glob(app_pattern) if app_matches: app_dir = app_matches[0] @@ -147,6 +150,9 @@ def postprocess(i): if os.path.isfile(geekbench_bin): os.chmod(geekbench_bin, 0o755) + if not os.path.exists(geekbench_dir): + return {'return': 1, 'error': f'{geekbench_dir} is not existing'} + env['MLC_GEEKBENCH_BIN_WITH_PATH'] = geekbench_bin env['MLC_GEEKBENCH_INSTALLED_PATH'] = geekbench_dir env['MLC_GEEKBENCH_VERSION'] = need_version diff --git a/script/get-geekbench/meta.yaml b/script/get-geekbench/meta.yaml index cec660ea5..d770d22b4 100644 --- a/script/get-geekbench/meta.yaml +++ b/script/get-geekbench/meta.yaml @@ -10,7 +10,6 @@ deps: - tags: detect,os - tags: detect,cpu input_mapping: - version: MLC_VERSION local_file: MLC_GEEKBENCH_LOCAL_FILE new_env_keys: - MLC_GEEKBENCH_* @@ -21,6 +20,9 @@ prehook_deps: - enable_if_env: MLC_GEEKBENCH_NEED_EXTRACT: - 'yes' + skip_if_env: + MLC_GEEKBENCH_INSTALLED_PATH: + - on env: MLC_DAE_EXTRACT_DOWNLOADED: 'yes' MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_GEEKBENCH_DOWNLOAD_PATH @@ -37,6 +39,9 @@ prehook_deps: - enable_if_env: MLC_GEEKBENCH_NEED_EXTRACT: - 'no' + skip_if_env: + MLC_GEEKBENCH_INSTALLED_PATH: + - on env: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_GEEKBENCH_DOWNLOAD_PATH extra_cache_tags: geekbench @@ -65,3 +70,8 @@ versions: 6.6.0: env: MLC_GEEKBENCH_VERSION_MAJOR: '6' +variations: + path.#: + env: + MLC_GEEKBENCH_INSTALLED_PATH: '#' + default_version: '' diff --git a/script/get-geekbench/run.sh b/script/get-geekbench/run.sh index 51bc1d668..2d69ccbaa 100755 --- a/script/get-geekbench/run.sh +++ b/script/get-geekbench/run.sh @@ -4,4 +4,3 @@ # Download and extraction is handled by the download-and-extract prehook dependency. # Path setup is handled in postprocess() of customize.py. -echo "Geekbench download and extraction completed via download-and-extract dependency." diff --git a/script/get-generic-sys-util/meta.yaml b/script/get-generic-sys-util/meta.yaml index 6f68021d3..d535c4e88 100644 --- a/script/get-generic-sys-util/meta.yaml +++ b/script/get-generic-sys-util/meta.yaml @@ -694,6 +694,20 @@ variations: brew: '' dnf: ntpdate yum: ntpdate + nasm: + env: + MLC_SYS_UTIL_NAME: nasm + MLC_SYS_UTIL_VERSION_CMD: nasm --version + MLC_SYS_UTIL_VERSION_RE: NASM version ([\d.]+) + MLC_TMP_VERSION_DETECT_GROUP_NUMBER: 0 + new_env_keys: + - MLC_NASM_VERSION + state: + nasm: + apt: nasm + brew: nasm + dnf: nasm + yum: nasm crossbuild-essential-arm64: env: MLC_SYS_UTIL_NAME: crossbuild-essential-arm64 From 518578b52fd8bc073634d19cc8fc730b787974c5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 20 Mar 2026 15:37:21 +0000 Subject: [PATCH 48/53] [Automated Commit] Document script/get-geekbench/meta.yaml [skip ci] --- script/get-geekbench/README.md | 68 ++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 script/get-geekbench/README.md diff --git a/script/get-geekbench/README.md b/script/get-geekbench/README.md new file mode 100644 index 000000000..bce677650 --- /dev/null +++ b/script/get-geekbench/README.md @@ -0,0 +1,68 @@ +# README for get-geekbench +This README is automatically generated. Create and add custom content in info.md. Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,geekbench,get-geekbench +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--local_file` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--search_folder_path` | The folder path where executables of a given script need to be searched. Search is done recursively upto 4 levels. | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--skip_sudo` | Skip SUDO detection | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Ungrouped + +- `path.#` _(# can be substituted dynamically)_ From 12bd1241603bafc4ee27c6feac55ab2334760b27 Mon Sep 17 00:00:00 2001 From: Arjun Date: Sun, 22 Mar 2026 11:28:15 +0000 Subject: [PATCH 49/53] Round geekbench mean scores to 3 decimal places, Windows run.bat fixes --- script/benchmark-program-geekbench/customize.py | 2 +- script/get-geekbench/run.bat | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/script/benchmark-program-geekbench/customize.py b/script/benchmark-program-geekbench/customize.py index 186ee28c9..c6eb376f1 100644 --- a/script/benchmark-program-geekbench/customize.py +++ b/script/benchmark-program-geekbench/customize.py @@ -97,7 +97,7 @@ def compute_stats(values): """Compute mean, median, olympic, cv_percent, min, max for a list of numeric values.""" if not values: return {} - mean_val = round(statistics.mean(values), 2) + mean_val = round(statistics.mean(values), 3) stdev_val = round(statistics.stdev(values), 2) if len(values) >= 2 else 0 result = { 'mean': mean_val, diff --git a/script/get-geekbench/run.bat b/script/get-geekbench/run.bat index 701b36c11..d4090a61e 100644 --- a/script/get-geekbench/run.bat +++ b/script/get-geekbench/run.bat @@ -5,6 +5,10 @@ rem get-geekbench run script (Windows) rem Download is handled by the download-and-extract prehook dependency. rem This script runs the setup installer silently and exports the installed path. +if not "%MLC_GEEKBENCH_INSTALLED_PATH%" == "" ( + exit /b 0 +) + if "%MLC_GEEKBENCH_DOWNLOAD_PATH%" == "" ( echo MLC_GEEKBENCH_DOWNLOAD_PATH is not set exit /b 1 From 213080bd5a00651f557bf59200759b6516cd8d1e Mon Sep 17 00:00:00 2001 From: Arjun Date: Mon, 23 Mar 2026 23:44:00 +0530 Subject: [PATCH 50/53] Support versions for lib-jemalloc --- script/README.md | 2 +- script/get-lib-jemalloc/meta.yaml | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/script/README.md b/script/README.md index 131b03259..a14673ec9 100644 --- a/script/README.md +++ b/script/README.md @@ -1,6 +1,6 @@ # MLCommons Automation Scripts -*Last updated: 2026-03-20 17:33:41* +*Last updated: 2026-03-23 23:44:00* This directory contains automation scripts for MLPerf benchmarks, AI/ML workflows, and development operations. diff --git a/script/get-lib-jemalloc/meta.yaml b/script/get-lib-jemalloc/meta.yaml index ce0166dd8..789020a35 100644 --- a/script/get-lib-jemalloc/meta.yaml +++ b/script/get-lib-jemalloc/meta.yaml @@ -19,6 +19,8 @@ deps: - MLC_GIT_SHA _submodules.: - MLC_GIT_SUBMODULES + _tag.: + - MLC_GIT_CHECKOUT_TAG names: - jemalloc-repo extra_cache_tags: jemalloc,repo,jemalloc-repo @@ -52,17 +54,20 @@ variations: version.official: base: - url.official + version.#: + env: + MLC_GIT_CHECKOUT_TAG: '#' url.#: - group: version + group: repo env: MLC_GIT_URL: '#' url.official: - group: version + group: repo env: MLC_GIT_URL: https://github.com/jemalloc/jemalloc.git MLC_JEMALLOC_VERSION_PREFIX: 'jemalloc-dev' url.facebook: - group: version + group: repo default: true env: MLC_GIT_URL: https://github.com/facebook/jemalloc.git From 7ae0e9958b596089ccbb458f3af3110e74ed89d9 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 23 Mar 2026 23:51:14 +0530 Subject: [PATCH 51/53] Fix model path for docker run (#868) --- script/app-mlperf-inference/meta.yaml | 2 +- .../customize.py | 5 +++-- script/get-ml-model-yolov11/customize.py | 5 +++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index 03787fdad..fb2dbd077 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -1720,7 +1720,7 @@ variations: - "yes" mounts: - "${{ MLC_ML_MODEL_YOLOV11_PATH }}:${{ MLC_ML_MODEL_YOLOV11_PATH }}" - - "${{ MLC_DATASET_YOLO_COCO2017_FILTERED_PATH }}:${{ MLC_DATASET_YOLO_COCO2017_FILTERED_PATH }}" + - "${{ MLC_ML_DATASET_MLPERF_INFERENCE_YOLO_COCO2017_FILTERED_DATASET_PATH }}:${{ MLC_ML_DATASET_MLPERF_INFERENCE_YOLO_COCO2017_FILTERED_DATASET_PATH }}" posthook_deps: - enable_if_env: MLC_MLPERF_LOADGEN_MODE: diff --git a/script/get-dataset-mlperf-inference-yolo-coco2017-filtered-dataset/customize.py b/script/get-dataset-mlperf-inference-yolo-coco2017-filtered-dataset/customize.py index 615fac7a9..e649b9a63 100644 --- a/script/get-dataset-mlperf-inference-yolo-coco2017-filtered-dataset/customize.py +++ b/script/get-dataset-mlperf-inference-yolo-coco2017-filtered-dataset/customize.py @@ -24,8 +24,9 @@ def postprocess(i): env = i['env'] if env.get('MLC_DOWNLOAD_MODE', '') != 'dry': - env['MLC_ML_DATASET_MLPERF_INFERENCE_YOLO_COCO2017_FILTERED_DATASET_PATH'] = os.path.join( - env['MLC_ML_DATASET_MLPERF_INFERENCE_YOLO_COCO2017_FILTERED_DATASET_PATH'], "val2017_safe") + if env.get('MLC_TMP_REQUIRE_DOWNLOAD', '') == "yes": + env['MLC_ML_DATASET_MLPERF_INFERENCE_YOLO_COCO2017_FILTERED_DATASET_PATH'] = os.path.join( + env['MLC_ML_DATASET_MLPERF_INFERENCE_YOLO_COCO2017_FILTERED_DATASET_PATH'], "val2017_safe") env['MLC_ML_DATASET_FILE_WITH_PATH'] = env['MLC_ML_DATASET_MLPERF_INFERENCE_YOLO_COCO2017_FILTERED_DATASET_PATH'] env['MLC_ML_DATASET_MLPERF_INFERENCE_YOLO_COCO2017_FILTERED_DATASET_ANNOTATION_PATH'] = os.path.join( env['MLC_ML_DATASET_FILE_WITH_PATH'], "annotations", "instances_val2017.json") diff --git a/script/get-ml-model-yolov11/customize.py b/script/get-ml-model-yolov11/customize.py index 9561190e3..0559f3620 100644 --- a/script/get-ml-model-yolov11/customize.py +++ b/script/get-ml-model-yolov11/customize.py @@ -23,8 +23,9 @@ def postprocess(i): env = i['env'] if env.get('MLC_DOWNLOAD_MODE', '') != 'dry': - env['MLC_ML_MODEL_YOLOV11_PATH'] = os.path.join( - env['MLC_ML_MODEL_YOLOV11_PATH'], 'yolo11l.pt') + if env.get('MLC_TMP_REQUIRE_DOWNLOAD') == "yes": + env['MLC_ML_MODEL_YOLOV11_PATH'] = os.path.join( + env['MLC_ML_MODEL_YOLOV11_PATH'], 'yolo11l.pt') env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['MLC_ML_MODEL_YOLOV11_PATH'] return {'return': 0} From a9468fb1712dd2eb82f5f1768c63c329ded42c00 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 23 Mar 2026 18:22:39 +0000 Subject: [PATCH 52/53] [Automated Commit] Document script/get-lib-jemalloc/meta.yaml [skip ci] --- script/get-lib-jemalloc/README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/script/get-lib-jemalloc/README.md b/script/get-lib-jemalloc/README.md index e5c0a1d0c..388594308 100644 --- a/script/get-lib-jemalloc/README.md +++ b/script/get-lib-jemalloc/README.md @@ -59,6 +59,12 @@ No script specific inputs | `--verify_ssl` | Verify SSL | | `False` | ## Variations +### Repo + +- `url.#` _(# can be substituted dynamically)_ +- `url.facebook` (default) +- `url.official` + ### Ungrouped - `branch.#` _(# can be substituted dynamically)_ @@ -69,10 +75,5 @@ No script specific inputs - `lg-page.#` _(# can be substituted dynamically)_ - `lg-quantum.#` _(# can be substituted dynamically)_ - `sha.#` _(# can be substituted dynamically)_ +- `version.#` _(# can be substituted dynamically)_ - `version.official` (base: url.official) - -### Version - -- `url.#` _(# can be substituted dynamically)_ -- `url.facebook` (default) -- `url.official` From 96088819b3cfd250f1b43b5532303a997ec3f7a1 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 25 Mar 2026 16:19:11 +0530 Subject: [PATCH 53/53] Enable yolo GH action --- .../test-mlperf-inference-yolo-closed-div.yml | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-yolo-closed-div.yml b/.github/workflows/test-mlperf-inference-yolo-closed-div.yml index d4677690a..a54e9ad1e 100644 --- a/.github/workflows/test-mlperf-inference-yolo-closed-div.yml +++ b/.github/workflows/test-mlperf-inference-yolo-closed-div.yml @@ -6,7 +6,7 @@ on: schedule: - cron: '0 0 * * *' # Runs daily at 12 AM UTC pull_request_target: - branches: [ "main_off", "dev_off" ] + branches: [ "main", "dev" ] paths: - '.github/workflows/test-mlperf-inference-yolo.yml' - '**' @@ -17,7 +17,7 @@ jobs: runs-on: ${{ matrix.os }} env: MLC_INDEX: "on" - SUBMISSION_DIR: ${{ github.workspace }}/mlperf_inference_results + RESULTS_DIR: ${{ github.workspace }}/mlperf_inference_results strategy: fail-fast: false matrix: @@ -51,20 +51,22 @@ jobs: - name: Test MLPerf Inference YOLO-v11 (Linux/macOS) run: | mlcr run-mlperf,inference,_full,_find-performance,_all-scenarios,_r6.0-dev --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }}x86" --model=yolo-99 --implementation=reference --category=edge --backend=${{ matrix.backend }} --framework=pytorch --device=cpu --execution_mode=test -v --quiet - mlcr run-mlperf,inference,_submission,_full,_all-modes,_all-scenarios,_r6.0-dev --division=closed --submission_dir=${{ env.SUBMISSION_DIR }} --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }}x86" --model=yolo-99 --implementation=reference --category=edge --backend=${{ matrix.backend }} --framework=pytorch --device=cpu --execution_mode=valid --multistream_target_latency=900 --env.MLC_MLPERF_USE_MAX_DURATION=no -v --quiet - + mlcr run-mlperf,inference,_compliance,_full,_all-modes,_all-scenarios,_r6.0-dev --division=closed --results_dir=${{ env.RESULTS_DIR }} --model=yolo-99 --implementation=reference --category=edge --backend=${{ matrix.backend }} --framework=pytorch --device=cpu --execution_mode=valid --multistream_target_latency=900 --env.MLC_MLPERF_USE_MAX_DURATION=no -v --quiet --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }}_python${{ matrix.python-version }}x86" - name: upload results artifact uses: actions/upload-artifact@v4 with: name: mlperf-inference-yolo-results-${{ matrix.os }}-py${{ matrix.python-version }}-bk${{ matrix.backend }} - path: ${{ env.SUBMISSION_DIR }} + path: ${{ env.RESULTS_DIR }} upload-results-to-github: needs: mlc-run runs-on: ubuntu-latest env: MLC_INDEX: "on" - SUBMISSION_DIR: ${{ github.workspace }}/mlperf_inference_results + SUBMISSION_DIR: ${{ github.workspace }}/mlperf_inference_submission + ARTIFACT_DOWNLOAD_PATH: ${{ github.workspace }}/gh_artifacts + RESULTS_DIR: ${{ github.workspace }}/mlperf_inference_results + concurrency: group: upload-results-v6.0 cancel-in-progress: false @@ -100,7 +102,7 @@ jobs: - name: Download benchmark artifacts uses: actions/download-artifact@v4 with: - path: "${{ env.SUBMISSION_DIR }}/closed" + path: "${{ env.ARTIFACT_DOWNLOAD_PATH }}" - name: Load secrets id: op-load-secrets @@ -109,6 +111,26 @@ jobs: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential + - name: Prepare Results Directory + run: | + set -euo pipefail + + mkdir -p "${{ env.RESULTS_DIR }}" + + # Iterate over system folders + for system_dir in "${{ env.ARTIFACT_DOWNLOAD_PATH }}/*"; do + if [ -d "${system_dir}/valid_results" ]; then + echo "Processing ${system_dir}/valid_results" + cp -r "${system_dir}/valid_results/"* "${{ env.RESULTS_DIR }}/" + else + echo "Skipping ${system_dir} (no valid_results directory)" + fi + done + + tree -a "${{ env.RESULTS_DIR }}" + + mlcr generate,inference,submission --adr.compiler.tags=gcc --version=v6.0 --clean --preprocess_submission=yes --submission_base_dir="${{ env.SUBMISSION_DIR}}" --results_dir="${{ env.RESULTS_DIR }}" --run_checker --submitter=MLCommons --tar=yes --division=closed --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --extra_checker_args="--skip-extra-accuracy-files-check" --quiet $docker_tags + - name: Push Results env: GITHUB_TOKEN: ${{ steps.op-load-secrets.outputs.PAT }}