Merge branch 'master' into submission_checker_refactor

Author: arjunsuresh

.github/workflows/auto-update-dev.yml

@@ -12,6 +12,7 @@ jobs:
     strategy:
       matrix: 
         branch: [ "dev", "docs" ]
+        
     runs-on: ubuntu-latest
 
     permissions:

.github/workflows/test-resnet50.yml

@@ -35,9 +35,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python3 -m pip install mlcflow
-        mlc pull repo mlcommons@mlperf-automations --branch=sub-gen-change
-      # python3 -m pip install mlc-scripts
+        python3 -m pip install mlc-scripts
     - name: Test Resnet50 and end to end submission generation
       run: |
-        mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --adr.inference-src.tags=_branch.submission_dir --submitter="MLCommons" --hw_name=default --model=resnet50 --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --adr.inference-src.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src.version=custom --adr.inference-src-loadgen.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom ${{ matrix.loadgen-flag }}
+        mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=resnet50 --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --adr.inference-src.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src.version=custom --adr.inference-src-loadgen.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom ${{ matrix.loadgen-flag }}

.github/workflows/test-retinanet.yml

@@ -33,9 +33,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python3 -m pip install mlcflow
-        mlc pull repo mlcommons@mlperf-automations --branch=sub-gen-change
-      # python3 -m pip install mlc-scripts
+        python3 -m pip install mlc-scripts
     - name: Test Retinanet and end to end submission generation
       run: |
-        mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --adr.inference-src.tags=_branch.submission_dir --quiet --submitter="MLCommons" --hw_name=default --model=retinanet --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=10 --adr.compiler.tags=gcc  --adr.inference-src.version=custom --adr.inference-src.tags=_repo.${{ github.event.pull_request.head.repo.html_url }},_branch.${{ github.event.pull_request.head.ref }} --adr.inference-src-loadgen.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom
+        mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=retinanet --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=10 --adr.compiler.tags=gcc  --adr.inference-src.version=custom --adr.inference-src.tags=_repo.${{ github.event.pull_request.head.repo.html_url }},_branch.${{ github.event.pull_request.head.ref }} --adr.inference-src-loadgen.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom

.github/workflows/test-rgat.yml

@@ -32,9 +32,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python3 -m pip install mlcflow
-        mlc pull repo mlcommons@mlperf-automations --branch=sub-gen-change
-      # python3 -m pip install mlc-scripts
+        python3 -m pip install mlc-scripts
     - name: Test R-GAT and end to end submission generation
       run: |
-        mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --adr.inference-src.tags=_branch.submission_dir --quiet --submitter="MLCommons" --category=datacenter --hw_name=default --model=rgat --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --adr.inference-src.tags=_branch.$PR_HEAD_REF,_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.tags=_branch.$PR_HEAD_REF,_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom ${{ matrix.loadgen-flag }}
+        mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --category=datacenter --hw_name=default --model=rgat --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --adr.inference-src.tags=_branch.$PR_HEAD_REF,_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.tags=_branch.$PR_HEAD_REF,_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom ${{ matrix.loadgen-flag }}

loadgen/README_BUILD.md

@@ -18,9 +18,9 @@ This will fetch the loadgen source, build and install the loadgen as a python mo
 
 Alternatively, we provide wheels for several python versions and operating system that can be installed using pip directly.
 
-    pip install mlperf-loadgen
+    pip install mlcommons-loadgen
 
-**NOTE:** Take into account that we only update the published wheels after an official release, they may not include the latest changes.
+**NOTE:** The `mlcommons-loadgen` PyPI package is automatically updated whenever the `loadgen/` folder in the `master` branch changes or when a release is published, as controlled by the workflow defined [here](https://github.com/mlcommons/inference/blob/master/.github/workflows/build_wheels.yml).
 
 ### Testing your Installation
 The following command will run a simple end-to-end demo:

loadgen/VERSION.txt

@@ -1 +1 @@
-5.1.1
+5.1.2

tools/submission/preprocess_submission.py

@@ -99,7 +99,7 @@ def delete_empty_dirs(src):
     return False
 
 
-def copy_submission_dir(src, dst, filter_submitter, keep_structure = True):
+def copy_submission_dir(src, dst, filter_submitter, keep_structure=True):
     """
     Copies the submission tree to output directory for processing
     """
@@ -116,15 +116,18 @@ def copy_submission_dir(src, dst, filter_submitter, keep_structure = True):
                 )
             else:
                 for dir in os.listdir(os.path.join(src, division, submitter)):
-                    if os.path.isdir(os.path.join(src, division, submitter, dir)):
-                        target_dir = "results" if dir in ["compliance", "measurements"] else dir
+                    if os.path.isdir(os.path.join(
+                            src, division, submitter, dir)):
+                        target_dir = "results" if dir in [
+                            "compliance", "measurements"] else dir
                         shutil.copytree(
                             os.path.join(src, division, submitter, dir),
                             os.path.join(dst, division, submitter, target_dir),
-                            dirs_exist_ok = True
+                            dirs_exist_ok=True
                         )
                 for file in os.listdir(os.path.join(src, division, submitter)):
-                    if os.path.isfile(os.path.join(src, division, submitter, file)):
+                    if os.path.isfile(os.path.join(
+                            src, division, submitter, file)):
                         shutil.copyfile(
                             os.path.join(src, division, submitter, file),
                             os.path.join(dst, division, submitter, file)
@@ -561,7 +564,11 @@ def main():
         log.error(f"output directory {args.output} already exists")
         sys.exit(1)
     os.makedirs(args.output)
-    copy_submission_dir(args.input, args.output, args.submitter, args.keep_structure)
+    copy_submission_dir(
+        args.input,
+        args.output,
+        args.submitter,
+        args.keep_structure)
     src_dir = args.output
 
     config = checker.Config(

tools/submission/submission_checker.py

@@ -1061,7 +1061,7 @@ def set_type(self, submission_type):
             self.optional = self.base["optional-scenarios-datacenter-edge"]
         else:
             raise ValueError("invalid system type")
-        
+
     def skip_calibration(self):
         return self.skip_calibration_check or self.version in ["v5.0"]
 
@@ -1893,7 +1893,7 @@ def get_power_metric(config, scenario_fixed, log_path, is_valid, res):
                 samples_per_query = 8
 
             if (scenario_fixed in ["MultiStream"]
-                ) and scenario in ["SingleStream"]:
+                    ) and scenario in ["SingleStream"]:
                 power_metric = (
                     avg_power * power_duration * samples_per_query * 1000 / num_queries
                 )
@@ -3040,7 +3040,6 @@ def check_measurement_dir(
             end = len(".json")
             break
 
-
     weight_data_types = None
     if system_file:
         with open(os.path.join(measurement_dir, system_file), "r") as f:

tools/submission/truncate_accuracy_log.py

@@ -172,7 +172,8 @@ def truncate_results_dir(filter_submitter, backup, scenarios_to_skip):
                                     acc_path, "accuracy.txt")
 
                                 # only TEST01 has an accuracy log
-                                if str(test).startswith("TEST") and test != "TEST01":
+                                if str(test).startswith(
+                                        "TEST") and test != "TEST01":
                                     continue
                                 if not os.path.exists(acc_log):
                                     log.error("%s missing", acc_log)