Merge pull request #647 from mlcommons/mlperf-inference

sync

Author: ctuning-admin

.github/workflows/build_wheel.yml

@@ -75,6 +75,7 @@ jobs:
         run: |
           git add VERSION
           git commit -m "Increment version to ${{ steps.do_version_increment.outputs.new_version }}"
+          git pull --rebase
           git push
 
       # Step 6: Update the git hash in the repository
@@ -83,6 +84,7 @@ jobs:
           python3 get_git_version.py > git_commit_hash.txt
           git add git_commit_hash.txt
           git commit -m "Updated git_commit_hash.txt"
+          git pull --rebase
           git push
 
       # Step 7: Install required dependencies

CHANGES.md

@@ -1,3 +1,6 @@
+### 20241123
+ [November 2024 release of cm4mlops](https://github.com/mlcommons/cm4mlops/discussions/590)
+
 ### 20240927
  * added "test dummy" script to test Docker containers
  * added more standard Nvidia Docker configuration for PyTorch

VERSION

@@ -1 +1 @@
-0.6.4
+0.6.6

automation/script/module.py

@@ -3526,7 +3526,9 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a
                     "update_tags_from_env_with_prefix", {})
                 for t in update_tags_from_env_with_prefix:
                     for key in update_tags_from_env_with_prefix[t]:
-                        if str(env.get(key, '')).strip() != '':
+                        if str(d.get('env', {}).get(key, '')).strip() != '':
+                            d['tags'] += "," + t + str(d.get('env')[key])
+                        elif str(env.get(key, '')).strip() != '':
                             d['tags'] += "," + t + str(env[key])
 
                 for key in clean_env_keys_deps:

git_commit_hash.txt

@@ -1 +1 @@
-f57fe4f69aa2d88c0c0ceb925a4a61723917c8e8
+e6fce27c346320bfab2e517ef82c3390531c4744

script/app-mlperf-inference-mlcommons-python/_cm.yaml

@@ -980,6 +980,9 @@ variations:
       - tags: get,generic-python-lib,_package.scipy
         names:
           - scipy
+      - tags: get,generic-python-lib,_package.pandas
+        names:
+          - pandas
 
   llama2-70b_:
     env:

script/get-dataset-mlperf-inference-igbh/_cm.yaml

@@ -31,10 +31,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,paper,node_feat
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL
@@ -46,10 +47,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,paper,node_label_19
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL
@@ -61,10 +63,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,paper,node_label_2K
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL
@@ -76,10 +79,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,paper,paper_id_index_mapping
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL
@@ -92,10 +96,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,paper_cites_paper,edge_index
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL
@@ -108,10 +113,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,author,author_id_index_mapping
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL
@@ -123,10 +129,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,author,node_feat
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL
@@ -139,10 +146,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,conference,conference_id_index_mapping
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL
@@ -154,10 +162,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,conference,node_feat
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL  
@@ -170,10 +179,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,institute,institute_id_index_mapping
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL
@@ -185,10 +195,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,institute,node_feat
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL
@@ -201,10 +212,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,journal,journal_id_index_mapping
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL
@@ -216,10 +228,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,journal,node_feat
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL
@@ -232,10 +245,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,fos,fos_id_index_mapping
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL
@@ -247,10 +261,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,fos,node_feat
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL
@@ -263,10 +278,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,author_affiliated_to_institute,edge_index
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL
@@ -279,10 +295,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,paper_published_journal,edge_index
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL
@@ -295,10 +312,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,paper_topic_fos,edge_index
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL
@@ -311,10 +329,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,paper_venue_conference,edge_index
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL
@@ -327,10 +346,11 @@ prehook_deps:
     extra_cache_tags: dataset,igbh,paper_written_by_author,edge_index
     force_cache: true
     enable_if_env:
-      CM_IGBH_DATASET_TYPE: full
+      CM_IGBH_DATASET_TYPE: 
+      - 'full'
     names:
     - dae
-    tags: download-and-extract
+    tags: download-and-extract,_wget
     update_tags_from_env_with_prefix:
       _url.:
         - CM_PACKAGE_URL

script/get-mlperf-inference-src/_cm.yaml

@@ -78,8 +78,6 @@ variations:
     base:
     - _branch.deepsparse
     - _repo.https://github.com/neuralmagic/inference
-    env:
-      CM_MLPERF_LAST_RELEASE: v4.0
   full-history:
     env:
       CM_GIT_DEPTH: ''
@@ -139,7 +137,7 @@ versions:
       CM_MLPERF_LAST_RELEASE: v4.1
   deepsparse:
     env:
-      CM_MLPERF_LAST_RELEASE: v4.0
+      CM_MLPERF_LAST_RELEASE: v4.1
       CM_TMP_GIT_CHECKOUT: deepsparse
       CM_TMP_GIT_URL: https://github.com/neuralmagic/inference
   main: