Skip to content

Commit de40c22

Browse files
authored
Merge pull request #810 from mlcommons/dev
Merge Dev
2 parents f2a32fa + 27d53d8 commit de40c22

File tree

26 files changed

+732
-122
lines changed

26 files changed

+732
-122
lines changed

.github/workflows/test-mlperf-inference-retinanet.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ name: MLPerf inference retinanet
22

33
on:
44
pull_request_target:
5-
branches: [ "main", "dev" ]
5+
branches: [ "main_off", "dev_off" ]
66
paths:
77
- '.github/workflows/test-mlperf-inference-retinanet.yml'
88
- '**'
Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
name: MLPerf inference YOLO-v11 Closed Division
2+
permissions:
3+
contents: read
4+
5+
on:
6+
schedule:
7+
- cron: '0 0 * * *' # Runs daily at 12 AM UTC
8+
pull_request_target:
9+
branches: [ "main_off", "dev_off" ]
10+
paths:
11+
- '.github/workflows/test-mlperf-inference-yolo.yml'
12+
- '**'
13+
- '!**.md'
14+
15+
jobs:
16+
mlc-run:
17+
runs-on: ${{ matrix.os }}
18+
env:
19+
MLC_INDEX: "on"
20+
SUBMISSION_DIR: ${{ github.workspace }}/mlperf_inference_results
21+
strategy:
22+
fail-fast: false
23+
matrix:
24+
os: [ubuntu-latest]
25+
python-version: [ "3.13", "3.12" ]
26+
backend: [ "pytorch" ]
27+
division: [ "open" ]
28+
29+
steps:
30+
- uses: actions/checkout@v4
31+
with:
32+
fetch-depth: 0
33+
34+
- name: Set up Python ${{ matrix.python-version }}
35+
uses: actions/setup-python@v3
36+
with:
37+
python-version: ${{ matrix.python-version }}
38+
39+
- name: Install mlcflow
40+
run: |
41+
pip install mlcflow
42+
pip install tabulate
43+
44+
- name: Pull MLOps repo
45+
shell: bash
46+
env:
47+
REPO: ${{ github.event.pull_request.head.repo.html_url }}
48+
BRANCH: ${{ github.event.pull_request.head.ref }}
49+
run: |
50+
mlc pull repo "$REPO" --branch="$BRANCH"
51+
- name: Test MLPerf Inference YOLO-v11 (Linux/macOS)
52+
run: |
53+
mlcr run-mlperf,inference,_full,_find-performance,_all-scenarios,_r6.0-dev --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }}x86" --model=yolo-99 --implementation=reference --category=edge --backend=${{ matrix.backend }} --framework=pytorch --device=cpu --execution_mode=test -v --quiet
54+
mlcr run-mlperf,inference,_submission,_full,_all-modes,_all-scenarios,_r6.0-dev --division=closed --submission_dir=${{ env.SUBMISSION_DIR }} --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }}x86" --model=yolo-99 --implementation=reference --category=edge --backend=${{ matrix.backend }} --framework=pytorch --device=cpu --execution_mode=valid --multistream_target_latency=900 --env.MLC_MLPERF_USE_MAX_DURATION=no -v --quiet
55+
56+
- name: upload results artifact
57+
uses: actions/upload-artifact@v4
58+
with:
59+
name: mlperf-inference-yolo-results-${{ matrix.os }}-py${{ matrix.python-version }}-bk${{ matrix.backend }}
60+
path: ${{ env.SUBMISSION_DIR }}
61+
62+
upload-results-to-github:
63+
needs: mlc-run
64+
runs-on: ubuntu-latest
65+
env:
66+
MLC_INDEX: "on"
67+
SUBMISSION_DIR: ${{ github.workspace }}/mlperf_inference_results
68+
concurrency:
69+
group: upload-results-v6.0
70+
cancel-in-progress: false
71+
strategy:
72+
fail-fast: false
73+
matrix:
74+
os: [ubuntu-latest]
75+
python-version: [ "3.13" ]
76+
77+
steps:
78+
- uses: actions/checkout@v4
79+
with:
80+
fetch-depth: 0
81+
82+
- name: Set up Python ${{ matrix.python-version }}
83+
uses: actions/setup-python@v3
84+
with:
85+
python-version: ${{ matrix.python-version }}
86+
87+
- name: Install mlcflow
88+
run: |
89+
pip install mlcflow
90+
pip install tabulate
91+
92+
- name: Pull MLOps repo
93+
shell: bash
94+
env:
95+
REPO: ${{ github.event.pull_request.head.repo.html_url }}
96+
BRANCH: ${{ github.event.pull_request.head.ref }}
97+
run: |
98+
mlc pull repo "$REPO" --branch="$BRANCH"
99+
100+
- name: Download benchmark artifacts
101+
uses: actions/download-artifact@v4
102+
with:
103+
path: "${{ env.SUBMISSION_DIR }}/closed"
104+
105+
- name: Load secrets
106+
id: op-load-secrets
107+
uses: 1password/load-secrets-action@v3
108+
env:
109+
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
110+
PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential
111+
112+
- name: Push Results
113+
env:
114+
GITHUB_TOKEN: ${{ steps.op-load-secrets.outputs.PAT }}
115+
if: github.repository_owner == 'mlcommons'
116+
run: |
117+
git config --global user.name "mlcommons-bot"
118+
git config --global user.email "mlcommons-bot@users.noreply.github.com"
119+
git config --global credential.https://github.com.helper ""
120+
git config --global credential.https://github.com.helper "!gh auth git-credential"
121+
git config --global credential.https://gist.github.com.helper ""
122+
git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
123+
mlcr push,github,mlperf,inference,submission --submission_dir=${{ env.SUBMISSION_DIR }} --repo_url=https://github.com/mlcommons/mlperf_inference_unofficial_submissions_v5.0/ --repo_branch=v6.0 --commit_message="Results from yolo-v11 GH action on ${{ matrix.os }}" --quiet
124+

.github/workflows/test-mlperf-inference-yolo.yml renamed to .github/workflows/test-mlperf-inference-yolo-open-div.yml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ on:
66
schedule:
77
- cron: '0 0 * * *' # Runs daily at 12 AM UTC
88
pull_request_target:
9-
branches: [ "main", "dev" ]
9+
branches: [ "main_off", "dev_off" ]
1010
paths:
1111
- '.github/workflows/test-mlperf-inference-yolo.yml'
1212
- '**'
@@ -50,13 +50,13 @@ jobs:
5050
mlc pull repo "$REPO" --branch="$BRANCH"
5151
- name: Test MLPerf Inference YOLO-v11 (Linux/macOS)
5252
run: |
53-
mlcr run-mlperf,inference,_full,_find-performance,_all-scenarios,_r6.0-dev --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }}x86" --model=yolo-99 --implementation=reference --category=edge --backend=${{ matrix.backend }} --framework=pytorch --device=cpu --execution_mode=test -adr.inference-src.tags=_branch.anandhu-eng-patch-13 --adr.inference-src-loadgen.tags=_branch.anandhu-eng-patch-13 --adr.inference-src.version=custom --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom -v --quiet
54-
mlcr run-mlperf,inference,_submission,_full,_all-modes,_all-scenarios,_r6.0-dev --submission_dir=${{ env.SUBMISSION_DIR }} --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }}x86" --model=yolo-99 --implementation=reference --category=edge --backend=${{ matrix.backend }} --framework=pytorch --device=cpu --execution_mode=valid -adr.inference-src.tags=_branch.anandhu-eng-patch-13 --adr.inference-src-loadgen.tags=_branch.anandhu-eng-patch-13 --adr.inference-src.version=custom --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom --multistream_target_latency=900 --env.MLC_MLPERF_USE_MAX_DURATION=no -v --quiet
53+
mlcr run-mlperf,inference,_full,_find-performance,_all-scenarios,_r6.0-dev --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }}x86" --model=yolo-99 --implementation=reference --category=edge --backend=${{ matrix.backend }} --framework=pytorch --device=cpu --execution_mode=test -v --quiet
54+
mlcr run-mlperf,inference,_submission,_full,_all-modes,_all-scenarios,_r6.0-dev --submission_dir=${{ env.SUBMISSION_DIR }} --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }}x86" --model=yolo-99 --implementation=reference --category=edge --backend=${{ matrix.backend }} --framework=pytorch --device=cpu --execution_mode=valid --multistream_target_latency=900 --env.MLC_MLPERF_USE_MAX_DURATION=no -v --quiet
5555
5656
- name: upload results artifact
5757
uses: actions/upload-artifact@v4
5858
with:
59-
name: ${{ matrix.division }}
59+
name: mlperf-inference-yolo-results-${{ matrix.os }}-py${{ matrix.python-version }}-bk${{ matrix.backend }}
6060
path: ${{ env.SUBMISSION_DIR }}
6161

6262
upload-results-to-github:
@@ -100,7 +100,7 @@ jobs:
100100
- name: Download benchmark artifacts
101101
uses: actions/download-artifact@v4
102102
with:
103-
path: ${{ env.SUBMISSION_DIR }}
103+
path: "${{ env.SUBMISSION_DIR }}/open"
104104

105105
- name: Load secrets
106106
id: op-load-secrets

README.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ At its core, MLCFlow relies on a single powerful automation, the Script, which i
3131
## 🤝 Contributing
3232
We welcome contributions from the community! To contribute:
3333
1. Submit pull requests (PRs) to the **`dev`** branch.
34-
2. Review our [CONTRIBUTORS.md](here) for guidelines and best practices.
34+
2. See [here](CONTRIBUTORS.md) for guidelines and best practices on contribution.
3535
3. Explore more about MLPerf Inference automation in the official [MLPerf Inference Documentation](https://docs.mlcommons.org/inference/).
3636

3737
Your contributions help drive the project forward!
@@ -61,6 +61,7 @@ This project is made possible through the generous support of:
6161
- [cKnowledge.org](https://cKnowledge.org)
6262
- [cTuning Foundation](https://cTuning.org)
6363
- [GATEOverflow](https://gateoverflow.in)
64+
- [AMD](https://www.amd.com)
6465
- [MLCommons](https://mlcommons.org)
6566

6667
We appreciate their contributions and sponsorship!

automation/script/cache_utils.py

Lines changed: 56 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ def prepare_cache_tags(i):
7777
cached_tags.append(x)
7878

7979
explicit_cached_tags = cached_tags.copy()
80+
explicit_cached_tags.append("-tmp")
8081

8182
# explicit variations
8283
if len(i['explicit_variation_tags']) > 0:
@@ -161,12 +162,15 @@ def search_cache(i, explicit_cached_tags):
161162
i['logger'].debug(
162163
i['recursion_spaces'] +
163164
' - Pruning cache list outputs with the following tags: {}'.format(explicit_cached_tags))
164-
165165
cache_list = i['cache_list']
166+
167+
n_tags = [p[1:] for p in explicit_cached_tags if p.startswith("-")]
168+
p_tags = [p for p in explicit_cached_tags if not p.startswith("-")]
169+
166170
pruned_cache_list = [
167171
item
168172
for item in cache_list
169-
if set(explicit_cached_tags) <= set(item.meta.get('tags', []))
173+
if set(p_tags) <= set(item.meta.get('tags', [])) and set(n_tags).isdisjoint(set(item.meta.get('tags', [])))
170174
]
171175

172176
return pruned_cache_list
@@ -214,9 +218,19 @@ def validate_cached_scripts(i, found_cached_scripts):
214218
'''
215219
valid = []
216220
if len(found_cached_scripts) > 0:
221+
222+
# We can consider doing quiet here if noise is too much
223+
# import logging
224+
# logger = i['logger']
225+
# logger_level_saved = logger.level
226+
# logger.setLevel(logging.ERROR)
227+
# saved_quiet = i['env'].get('MLC_QUIET', False)
228+
# i['env']['MLC_QUIET'] = True
217229
for cached_script in found_cached_scripts:
218230
if is_cached_entry_valid(i, cached_script):
219231
valid.append(cached_script)
232+
# logger.setLevel(logger_level_saved)
233+
# i['env']['MLC_QUIET'] = saved_quiet
220234

221235
return valid
222236

@@ -417,9 +431,7 @@ def find_cached_script(i):
417431
i, explicit_cached_tags, found_cached_scripts)
418432
found_cached_scripts = validate_cached_scripts(i, found_cached_scripts)
419433

420-
search_tags = '-tmp'
421-
if len(explicit_cached_tags) > 0:
422-
search_tags += ',' + ','.join(explicit_cached_tags)
434+
search_tags = ','.join(explicit_cached_tags)
423435

424436
return {'return': 0, 'cached_tags': cached_tags,
425437
'search_tags': search_tags, 'found_cached_scripts': found_cached_scripts}
@@ -430,40 +442,52 @@ def find_cached_script(i):
430442

431443
def fix_cache_paths(cached_path, env):
432444

433-
current_cache_path = cached_path
445+
current_cache_path = os.path.normpath(cached_path)
434446

435447
new_env = env # just a reference
436448

449+
def normalize_and_replace_path(path_str):
450+
"""Helper to normalize and replace cache paths in a string."""
451+
# Normalize the path to use the current OS separators
452+
normalized = os.path.normpath(path_str)
453+
454+
# Check if path contains local/cache or local\cache pattern
455+
path_parts = normalized.split(os.sep)
456+
457+
try:
458+
local_idx = path_parts.index("local")
459+
if local_idx + \
460+
1 < len(path_parts) and path_parts[local_idx + 1] == "cache":
461+
# Extract the loaded cache path (up to and including "cache")
462+
loaded_cache_path = os.sep.join(path_parts[:local_idx + 2])
463+
loaded_cache_path_norm = os.path.normpath(loaded_cache_path)
464+
465+
if loaded_cache_path_norm != current_cache_path and os.path.exists(
466+
current_cache_path):
467+
# Replace old cache path with current cache path
468+
return normalized.replace(
469+
loaded_cache_path_norm, current_cache_path)
470+
except (ValueError, IndexError):
471+
# "local" not in path or malformed path
472+
pass
473+
474+
return normalized
475+
437476
for key, val in new_env.items():
438-
# Check for a path separator in a string and determine the
439-
# separator
440-
if isinstance(val, str) and any(sep in val for sep in [
441-
"/local/cache/", "\\local\\cache\\"]):
442-
sep = "/" if "/local/cache/" in val else "\\"
443-
444-
path_split = val.split(sep)
445-
repo_entry_index = path_split.index("local")
446-
loaded_cache_path = sep.join(
447-
path_split[0:repo_entry_index + 2])
448-
if loaded_cache_path != current_cache_path and os.path.exists(
449-
current_cache_path):
450-
new_env[key] = val.replace(
451-
loaded_cache_path, current_cache_path).replace(sep, "/")
477+
if isinstance(val, str):
478+
# Check if path contains cache directory pattern
479+
normalized_val = val.replace('\\', os.sep).replace('/', os.sep)
480+
if os.sep.join(['local', 'cache']) in normalized_val:
481+
new_env[key] = normalize_and_replace_path(val)
452482

453483
elif isinstance(val, list):
454484
for i, val2 in enumerate(val):
455-
if isinstance(val2, str) and any(sep in val2 for sep in [
456-
"/local/cache/", "\\local\\cache\\"]):
457-
sep = "/" if "/local/cache/" in val2 else "\\"
458-
459-
path_split = val2.split(sep)
460-
repo_entry_index = path_split.index("local")
461-
loaded_cache_path = sep.join(
462-
path_split[0:repo_entry_index + 2])
463-
if loaded_cache_path != current_cache_path and os.path.exists(
464-
current_cache_path):
465-
new_env[key][i] = val2.replace(
466-
loaded_cache_path, current_cache_path).replace(sep, "/")
485+
if isinstance(val2, str):
486+
# Check if path contains cache directory pattern
487+
normalized_val2 = val2.replace(
488+
'\\', os.sep).replace('/', os.sep)
489+
if os.sep.join(['local', 'cache']) in normalized_val2:
490+
new_env[key][i] = normalize_and_replace_path(val2)
467491

468492
return {'return': 0, 'new_env': new_env}
469493

0 commit comments

Comments
 (0)