Merge Dev

.github/workflows/test-mlperf-inference-retinanet.yml

@@ -2,7 +2,7 @@ name: MLPerf inference retinanet
 
 on:
   pull_request_target:
-    branches: [ "main", "dev" ]
+    branches: [ "main_off", "dev_off" ]
     paths:
       - '.github/workflows/test-mlperf-inference-retinanet.yml'
       - '**'

.github/workflows/test-mlperf-inference-yolo-closed-div.yml

@@ -0,0 +1,124 @@
+name: MLPerf inference YOLO-v11 Closed Division
+permissions:
+  contents: read
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Runs daily at 12 AM UTC
+  pull_request_target:
+    branches: [ "main_off", "dev_off" ]
+    paths:
+      - '.github/workflows/test-mlperf-inference-yolo.yml'
+      - '**'
+      - '!**.md'
+
+jobs:
+  mlc-run:
+    runs-on: ${{ matrix.os }}
+    env:
+      MLC_INDEX: "on"
+      SUBMISSION_DIR: ${{ github.workspace }}/mlperf_inference_results
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: [ "3.13", "3.12" ]
+        backend: [ "pytorch" ]
+        division: [ "open" ]
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install mlcflow
+      run: |
+        pip install mlcflow
+        pip install tabulate
+
+    - name: Pull MLOps repo
+      shell: bash
+      env:
+        REPO: ${{ github.event.pull_request.head.repo.html_url }}
+        BRANCH: ${{ github.event.pull_request.head.ref }}
+      run: |
+        mlc pull repo "$REPO" --branch="$BRANCH"
+    - name: Test MLPerf Inference YOLO-v11 (Linux/macOS)
+      run: |
+        mlcr run-mlperf,inference,_full,_find-performance,_all-scenarios,_r6.0-dev --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }}x86" --model=yolo-99 --implementation=reference --category=edge --backend=${{ matrix.backend }} --framework=pytorch --device=cpu --execution_mode=test -v --quiet     
+        mlcr run-mlperf,inference,_submission,_full,_all-modes,_all-scenarios,_r6.0-dev --division=closed --submission_dir=${{ env.SUBMISSION_DIR }} --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }}x86" --model=yolo-99 --implementation=reference --category=edge --backend=${{ matrix.backend }} --framework=pytorch --device=cpu --execution_mode=valid --multistream_target_latency=900 --env.MLC_MLPERF_USE_MAX_DURATION=no -v --quiet     
+
+    - name: upload results artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: mlperf-inference-yolo-results-${{ matrix.os }}-py${{ matrix.python-version }}-bk${{ matrix.backend }}
+        path: ${{ env.SUBMISSION_DIR }}
+
+  upload-results-to-github:
+    needs: mlc-run
+    runs-on: ubuntu-latest
+    env:
+      MLC_INDEX: "on"
+      SUBMISSION_DIR: ${{ github.workspace }}/mlperf_inference_results
+    concurrency: 
+      group: upload-results-v6.0
+      cancel-in-progress: false
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: [ "3.13" ]
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install mlcflow
+      run: |
+        pip install mlcflow
+        pip install tabulate
+
+    - name: Pull MLOps repo
+      shell: bash
+      env:
+        REPO: ${{ github.event.pull_request.head.repo.html_url }}
+        BRANCH: ${{ github.event.pull_request.head.ref }}
+      run: |
+        mlc pull repo "$REPO" --branch="$BRANCH"
+
+    - name: Download benchmark artifacts
+      uses: actions/download-artifact@v4
+      with:
+        path: "${{ env.SUBMISSION_DIR }}/closed"
+
+    - name: Load secrets
+      id: op-load-secrets
+      uses: 1password/load-secrets-action@v3
+      env:
+        OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+        PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential
+
+    - name: Push Results
+      env:
+        GITHUB_TOKEN: ${{ steps.op-load-secrets.outputs.PAT }}
+      if: github.repository_owner == 'mlcommons'
+      run: |
+        git config --global user.name "mlcommons-bot"
+        git config --global user.email "mlcommons-bot@users.noreply.github.com"
+        git config --global credential.https://github.com.helper ""
+        git config --global credential.https://github.com.helper "!gh auth git-credential"
+        git config --global credential.https://gist.github.com.helper ""
+        git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
+        mlcr push,github,mlperf,inference,submission --submission_dir=${{ env.SUBMISSION_DIR }} --repo_url=https://github.com/mlcommons/mlperf_inference_unofficial_submissions_v5.0/ --repo_branch=v6.0 --commit_message="Results from yolo-v11 GH action on ${{ matrix.os }}" --quiet
+

.github/workflows/test-mlperf-inference-yolo.yml → .github/workflows/test-mlperf-inference-yolo-open-div.yml

@@ -6,7 +6,7 @@ on:
   schedule:
     - cron: '0 0 * * *'  # Runs daily at 12 AM UTC
   pull_request_target:
-    branches: [ "main", "dev" ]
+    branches: [ "main_off", "dev_off" ]
     paths:
       - '.github/workflows/test-mlperf-inference-yolo.yml'
       - '**'
@@ -50,13 +50,13 @@ jobs:
         mlc pull repo "$REPO" --branch="$BRANCH"
     - name: Test MLPerf Inference YOLO-v11 (Linux/macOS)
       run: |
-        mlcr run-mlperf,inference,_full,_find-performance,_all-scenarios,_r6.0-dev --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }}x86" --model=yolo-99 --implementation=reference --category=edge --backend=${{ matrix.backend }} --framework=pytorch --device=cpu --execution_mode=test -adr.inference-src.tags=_branch.anandhu-eng-patch-13 --adr.inference-src-loadgen.tags=_branch.anandhu-eng-patch-13 --adr.inference-src.version=custom --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom -v --quiet     
-        mlcr run-mlperf,inference,_submission,_full,_all-modes,_all-scenarios,_r6.0-dev --submission_dir=${{ env.SUBMISSION_DIR }} --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }}x86" --model=yolo-99 --implementation=reference --category=edge --backend=${{ matrix.backend }} --framework=pytorch --device=cpu --execution_mode=valid -adr.inference-src.tags=_branch.anandhu-eng-patch-13 --adr.inference-src-loadgen.tags=_branch.anandhu-eng-patch-13 --adr.inference-src.version=custom --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom --multistream_target_latency=900 --env.MLC_MLPERF_USE_MAX_DURATION=no -v --quiet     
+        mlcr run-mlperf,inference,_full,_find-performance,_all-scenarios,_r6.0-dev --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }}x86" --model=yolo-99 --implementation=reference --category=edge --backend=${{ matrix.backend }} --framework=pytorch --device=cpu --execution_mode=test -v --quiet     
+        mlcr run-mlperf,inference,_submission,_full,_all-modes,_all-scenarios,_r6.0-dev --submission_dir=${{ env.SUBMISSION_DIR }} --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }}x86" --model=yolo-99 --implementation=reference --category=edge --backend=${{ matrix.backend }} --framework=pytorch --device=cpu --execution_mode=valid --multistream_target_latency=900 --env.MLC_MLPERF_USE_MAX_DURATION=no -v --quiet     
 
     - name: upload results artifact
       uses: actions/upload-artifact@v4
       with:
-        name: ${{ matrix.division }}
+        name: mlperf-inference-yolo-results-${{ matrix.os }}-py${{ matrix.python-version }}-bk${{ matrix.backend }}
         path: ${{ env.SUBMISSION_DIR }}
 
   upload-results-to-github:
@@ -100,7 +100,7 @@ jobs:
     - name: Download benchmark artifacts
       uses: actions/download-artifact@v4
       with:
-        path: ${{ env.SUBMISSION_DIR }}
+        path: "${{ env.SUBMISSION_DIR }}/open"
 
     - name: Load secrets
       id: op-load-secrets

README.md

@@ -31,7 +31,7 @@ At its core, MLCFlow relies on a single powerful automation, the Script, which i
 ## 🤝 Contributing
 We welcome contributions from the community! To contribute:
 1. Submit pull requests (PRs) to the **`dev`** branch.
-2. Review our [CONTRIBUTORS.md](here) for guidelines and best practices.
+2. See [here](CONTRIBUTORS.md) for guidelines and best practices on contribution.
 3. Explore more about MLPerf Inference automation in the official [MLPerf Inference Documentation](https://docs.mlcommons.org/inference/).
 
 Your contributions help drive the project forward!
@@ -61,6 +61,7 @@ This project is made possible through the generous support of:
 - [cKnowledge.org](https://cKnowledge.org)
 - [cTuning Foundation](https://cTuning.org)
 - [GATEOverflow](https://gateoverflow.in)
+- [AMD](https://www.amd.com)
 - [MLCommons](https://mlcommons.org)
 
 We appreciate their contributions and sponsorship!

automation/script/cache_utils.py

@@ -77,6 +77,7 @@ def prepare_cache_tags(i):
                 cached_tags.append(x)
 
     explicit_cached_tags = cached_tags.copy()
+    explicit_cached_tags.append("-tmp")
 
     # explicit variations
     if len(i['explicit_variation_tags']) > 0:
@@ -161,12 +162,15 @@ def search_cache(i, explicit_cached_tags):
     i['logger'].debug(
         i['recursion_spaces'] +
         '    - Pruning cache list outputs with the following tags: {}'.format(explicit_cached_tags))
-
     cache_list = i['cache_list']
+
+    n_tags = [p[1:] for p in explicit_cached_tags if p.startswith("-")]
+    p_tags = [p for p in explicit_cached_tags if not p.startswith("-")]
+
     pruned_cache_list = [
         item
         for item in cache_list
-        if set(explicit_cached_tags) <= set(item.meta.get('tags', []))
+        if set(p_tags) <= set(item.meta.get('tags', [])) and set(n_tags).isdisjoint(set(item.meta.get('tags', [])))
     ]
 
     return pruned_cache_list
@@ -214,9 +218,19 @@ def validate_cached_scripts(i, found_cached_scripts):
     '''
     valid = []
     if len(found_cached_scripts) > 0:
+
+        # We can consider doing quiet here if noise is too much
+        # import logging
+        # logger = i['logger']
+        # logger_level_saved = logger.level
+        # logger.setLevel(logging.ERROR)
+        # saved_quiet = i['env'].get('MLC_QUIET', False)
+        # i['env']['MLC_QUIET'] = True
         for cached_script in found_cached_scripts:
             if is_cached_entry_valid(i, cached_script):
                 valid.append(cached_script)
+        # logger.setLevel(logger_level_saved)
+        # i['env']['MLC_QUIET'] = saved_quiet
 
     return valid
 
@@ -417,9 +431,7 @@ def find_cached_script(i):
         i, explicit_cached_tags, found_cached_scripts)
     found_cached_scripts = validate_cached_scripts(i, found_cached_scripts)
 
-    search_tags = '-tmp'
-    if len(explicit_cached_tags) > 0:
-        search_tags += ',' + ','.join(explicit_cached_tags)
+    search_tags = ','.join(explicit_cached_tags)
 
     return {'return': 0, 'cached_tags': cached_tags,
             'search_tags': search_tags, 'found_cached_scripts': found_cached_scripts}
@@ -430,40 +442,52 @@ def find_cached_script(i):
 
 def fix_cache_paths(cached_path, env):
 
-    current_cache_path = cached_path
+    current_cache_path = os.path.normpath(cached_path)
 
     new_env = env  # just a reference
 
+    def normalize_and_replace_path(path_str):
+        """Helper to normalize and replace cache paths in a string."""
+        # Normalize the path to use the current OS separators
+        normalized = os.path.normpath(path_str)
+
+        # Check if path contains local/cache or local\cache pattern
+        path_parts = normalized.split(os.sep)
+
+        try:
+            local_idx = path_parts.index("local")
+            if local_idx + \
+                    1 < len(path_parts) and path_parts[local_idx + 1] == "cache":
+                # Extract the loaded cache path (up to and including "cache")
+                loaded_cache_path = os.sep.join(path_parts[:local_idx + 2])
+                loaded_cache_path_norm = os.path.normpath(loaded_cache_path)
+
+                if loaded_cache_path_norm != current_cache_path and os.path.exists(
+                        current_cache_path):
+                    # Replace old cache path with current cache path
+                    return normalized.replace(
+                        loaded_cache_path_norm, current_cache_path)
+        except (ValueError, IndexError):
+            # "local" not in path or malformed path
+            pass
+
+        return normalized
+
     for key, val in new_env.items():
-        # Check for a path separator in a string and determine the
-        # separator
-        if isinstance(val, str) and any(sep in val for sep in [
-                "/local/cache/", "\\local\\cache\\"]):
-            sep = "/" if "/local/cache/" in val else "\\"
-
-            path_split = val.split(sep)
-            repo_entry_index = path_split.index("local")
-            loaded_cache_path = sep.join(
-                path_split[0:repo_entry_index + 2])
-            if loaded_cache_path != current_cache_path and os.path.exists(
-                    current_cache_path):
-                new_env[key] = val.replace(
-                    loaded_cache_path, current_cache_path).replace(sep, "/")
+        if isinstance(val, str):
+            # Check if path contains cache directory pattern
+            normalized_val = val.replace('\\', os.sep).replace('/', os.sep)
+            if os.sep.join(['local', 'cache']) in normalized_val:
+                new_env[key] = normalize_and_replace_path(val)
 
         elif isinstance(val, list):
             for i, val2 in enumerate(val):
-                if isinstance(val2, str) and any(sep in val2 for sep in [
-                        "/local/cache/", "\\local\\cache\\"]):
-                    sep = "/" if "/local/cache/" in val2 else "\\"
-
-                    path_split = val2.split(sep)
-                    repo_entry_index = path_split.index("local")
-                    loaded_cache_path = sep.join(
-                        path_split[0:repo_entry_index + 2])
-                    if loaded_cache_path != current_cache_path and os.path.exists(
-                            current_cache_path):
-                        new_env[key][i] = val2.replace(
-                            loaded_cache_path, current_cache_path).replace(sep, "/")
+                if isinstance(val2, str):
+                    # Check if path contains cache directory pattern
+                    normalized_val2 = val2.replace(
+                        '\\', os.sep).replace('/', os.sep)
+                    if os.sep.join(['local', 'cache']) in normalized_val2:
+                        new_env[key][i] = normalize_and_replace_path(val2)
 
     return {'return': 0, 'new_env': new_env}