Sync Dev

.github/pull_request_template.md

@@ -0,0 +1,26 @@
+## ✅ PR Checklist
+
+- [ ] Target branch is `dev`
+
+📌 Note: PRs must be raised against `dev`. Do not commit directly to `main`.
+
+### ✅ Testing & CI
+- [ ] Have tested the changes in my local environment, else have properly conveyed in the PR description
+- [ ] The change includes a GitHub Action to test the script(if it is possible to be added).
+- [ ] No existing GitHub Actions are failing because of this change.
+
+### 📚 Documentation
+- [ ] README or help docs are updated for new features or changes.
+- [ ] CLI help messages are meaningful and complete.
+
+### 📁 File Hygiene & Output Handling
+- [ ] No unintended files (e.g., logs, cache, temp files, __pycache__, output folders) are committed.
+
+### 🛡️ Safety & Security
+- [ ] No secrets or credentials are committed.
+- [ ] Paths, shell commands, and environment handling are safe and portable.
+
+### 🙌 Contribution Hygiene
+- [ ] PR title and description are concise and clearly state the purpose of the change.
+- [ ] Related issues (if any) are properly referenced using `Fixes #` or `Closes #`.
+- [ ] All reviewer feedback has been addressed.

.github/workflows/codeql.yml

@@ -0,0 +1,100 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL Advanced"
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  schedule:
+    - cron: '43 6 * * 0'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: actions
+          build-mode: none
+        - language: python
+          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'rust', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Add any setup steps before running the `github/codeql-action/init` action.
+    # This includes steps like installing compilers or runtimes (`actions/setup-node`
+    # or others). This is typically only required for manual builds.
+    # - name: Setup runtime (example)
+    #   uses: actions/setup-example@v1
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      shell: bash
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/mlperf-inference-bert.yml

@@ -2,7 +2,7 @@ name: MLPerf inference bert (deepsparse, tf, onnxruntime, pytorch)
 
 on:
   pull_request:
-    branches: [ "main", "dev" ]
+    branches: [ "main_off", "dev_off" ]
     paths:
       - '.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml'
       - '**'

VERSION

@@ -1 +1 @@
-1.0.8
+1.1.0

docs/install/index.md

@@ -48,8 +48,11 @@ This step is not mandatory. But the latest `pip` install requires this or else w
 
 === "Windows"
     ```bash
-    python3 -m venv mlcflow
+    python -m venv mlcflow
     mlcflow\Scripts\activate.bat
+    ```
+    Run as Administrator
+    ```bash
     git config --system core.longpaths true
     ```
 
@@ -60,9 +63,14 @@ If you are not using virtual ENV for installation, the latest `pip` install requ
 ```bash
 pip install mlcflow
 ```
-
+## Pull the Automation Repo
+```bash
+mlc pull repo mlcommons@mlperf-automations
+```
+* If you are forking https://github.com/mlcommons/mlperf-automations you can substitute the above command by `mlc pull repo <your_github_username>@mlperf-automations`.
+
 !!! tip
-    If you want to pull the latest changes (recommended), please do `mlc pull repo` after the installation.
+    If you want to pull the latest changes (recommended), please do `mlc pull repo` periodically.
 
 
 Now, you are ready to use the `mlc` commands. Currently, `mlc` is being used to automate the benchmark runs for:

docs/targets/script/execution-flow.md

@@ -36,14 +36,14 @@ When we run an MLC script we can also pass inputs to it and any input added in `
 We can use `skip_if_env` dictionary inside any `deps`, `prehook_deps`, `posthook_deps` or `post_deps` to make its execution conditional
 
 ### Versions
-We can specify any specific version of a script using `version`. `version_max` and `version_min` are also possible options. 
+We can specify any specific version of a script using `version`. `version_max`, `version_min` and `version_max_usable` are also possible options. 
 
-* When `version_min` is given, any version above this if present in the cache or detected in the system can be chosen. If nothing is detected `default_version` if present and if above `version_min` will be used for installation. Otherwise `version_min` will be used as `version`.
+* When `version_min` is given, any version above this if present in the cache or detected in the system can be chosen. If nothing is detected `default_version` if present and if above `version_min` will be used for installation. Otherwise `version_max_usable` if present or else `version_min` will be used as `version`.
 
 * When `version_max` is given, any version below this if present in the cache or detected in the system can be chosen. If nothing is detected `default_version` if present and if below `version_max` will be used for installation. Otherwise `version_max_usable` (additional needed input for `version_max`) will be used as `version`.
-
+  
 ### Variations
-* Variations are used to customize MLC script and each unique combination of variations uses a unique cache entry. Each variation can turn on `env` keys also any other meta including dependencies specific to it. Variations are turned on like tags but with a `_` prefix. For example, if a script is having tags `"get,myscript"`, to call the variation `"test"` inside it, we have to use tags `"get,myscript,_test"`. 
+* Variations are used to customize MLC script and each set of variations uses a unique cache entry. A variation can turn on `env` keys and any other meta including dependencies specific to it. Variations are turned on like tags but with a `_` prefix. For example, if a script is having tags `"get,myscript"`, to call the variation `"test"` inside it, we can use tags `"get,myscript,_test"`. 
 
 #### Variation groups
 `group` is a key to map variations into a group and at any time only one variation from a group can be used in the variation tags. For example, both `cpu` and `cuda` can be two variations under the `device` group, but user can at any time use either `cpu` or `cuda` as variation tags but not both.
@@ -63,6 +63,7 @@ Sometimes it is difficult to add all variations needed for a script like say `ba
 * Similarly we can avoid any env key from being passed to a given dependency by adding the prefix of the key in the `clean_env_keys` list of the concerned dependency.
 * `--input` is automatically converted to `MLC_INPUT` env key
 * `version` is converted to `MLC_VERSION`, ``version_min` to `MLC_VERSION_MIN` and `version_max` to `MLC_VERSION_MAX`
+* `mlc 
 * If `env['MLC_GH_TOKEN']=TOKEN_VALUE` is set then git URLs (specified by `MLC_GIT_URL`) are changed to add this token.
 * If `env['MLC_GIT_SSH']=yes`, then git URLs are changed to SSH from HTTPS.
 

mkdocs.yml

@@ -1,5 +1,6 @@
 site_name: MLCFlow Documentation
 repo_url: https://github.com/mlcommons/mlcflow
+site_url: https://docs.mlcommons.org/mlcflow
 theme:
   name: material
   logo: img/logo_v2.svg

mlc/action.py

@@ -23,7 +23,6 @@ class Action:
     logger = None
     local_repo = None
     current_repo_path = None
-    #mlc = None
     repos = [] #list of Repo objects
 
     # Main access function to simulate a Python interface for CLI
@@ -167,7 +166,7 @@ def load_repos(self):
 
 
     def __init__(self):        
-        setup_logging(log_path=os.getcwd(),log_file='mlc-log.txt')
+        setup_logging(log_path=os.getcwd(), log_file='.mlc-log.txt')
         self.logger = logger
 
         temp_repo = os.environ.get('MLC_REPOS','').strip()
@@ -654,6 +653,7 @@ def search(self, i):
         uid = i.get("uid")
         alias = i.get("alias")
         item_repo = i.get('item_repo')
+        exact_tags_match = i.get('exact_tags_match', False)
         fetch_all = True if i.get('fetch_all') else False
 
         # For targets like cache, sometimes user would need to clear the entire cache folder present in the system
@@ -707,13 +707,13 @@ def search(self, i):
                     for res in target_index:
                         if os.path.basename(res["path"]) == folder_name:
                             it = Item(res['path'], res['repo'])
-                            #result.append(it)
+                            result.append(it)
             else:
                 tags = i.get("tags")
                 if tags:
                     tags_split = tags.split(",")
                 else:
-                    return {"return":1, "error": f"Tags are not specifeid for completing the specific action"}
+                    return {"return":1, "error": f"Tags are not specified for completing the requested action"}
                 if target == "script":
                     non_variation_tags = [t for t in tags_split if not t.startswith("_")]
                     tags_to_match = non_variation_tags
@@ -726,7 +726,7 @@ def search(self, i):
                 p_tags = list(set(tags_to_match) - set(n_tags_))
                 for res in target_index:
                     c_tags = res["tags"]
-                    if set(p_tags).issubset(set(c_tags)) and set(n_tags).isdisjoint(set(c_tags)):
+                    if (exact_tags_match and set(p_tags) == set(c_tags)) or (not exact_tags_match and set(p_tags).issubset(set(c_tags)) and set(n_tags).isdisjoint(set(c_tags))):
                         it = Item(res['path'], res['repo'])
                         result.append(it)
         return {'return': 0, 'list': result}

mlc/logger.py

@@ -21,7 +21,7 @@ def format(self, record):
 
 
 # Set up logging configuration
-def setup_logging(log_path = os.getcwd(),log_file = 'mlc-log.txt'):
+def setup_logging(log_path = os.getcwd(), log_file = '.mlc-log.txt'):
 
     if not logger.hasHandlers():
         logFormatter = ColoredFormatter('[%(asctime)s %(filename)s:%(lineno)d %(levelname)s] - %(message)s')

mlc/main.py

@@ -64,11 +64,12 @@ def search(self, i):
                         uid = item_split[1]
                     else:
                         uid = item_split[0]
-            for res in target_index:
-                c_tags = res["tags"]
-                if set(p_tags).issubset(set(c_tags)) and set(n_tags).isdisjoint(set(c_tags)) and (not uid or uid == res['uid']) and (not alias or alias == res['alias']):
-                    it = Item(res['path'], res['repo'])
-                    result.append(it)
+            if tags or uid or i.get('all'):
+                for res in target_index:
+                    c_tags = res["tags"]
+                    if set(p_tags).issubset(set(c_tags)) and set(n_tags).isdisjoint(set(c_tags)) and (not uid or uid == res['uid']) and (not alias or alias == res['alias']):
+                        it = Item(res['path'], res['repo'])
+                        result.append(it)
         #logger.info(result)
         return {'return': 0, 'list': result}
         #indices
@@ -152,7 +153,7 @@ def main():
         action_parser.add_argument('extra', nargs=argparse.REMAINDER, help='Extra options (e.g.,  -v)')
 
     # Script specific subcommands
-    for action in ['docker', 'experiment']:
+    for action in ['docker', 'experiment', 'doc', 'lint']:
         action_parser = subparsers.add_parser(action, help=f'{action.capitalize()} a target.')
         action_parser.add_argument('target', choices=['script', 'run'], help='Target type (script).')
         # the argument given after target and before any extra options like --tags will be stored in "details"
@@ -247,7 +248,7 @@ def main():
         if args.target == "repo":
             run_args['repo'] = args.details
 
-    if args.command in ['docker', 'experiment']:
+    if args.command in ['docker', 'experiment', 'doc', 'lint']:
         if args.target == "run":
             run_args['target'] = 'script' #allowing run to be used for docker run instead of docker script
             args.target = "script"

mlc/repo_action.py

@@ -72,7 +72,7 @@ def add(self, run_args):
             return {"return": 1, "error": "The repository to be added is not specified"}
 
         i_repo_path = run_args['repo'] #can be a path, forder_name or URL
-        repo_folder_name = os.path.basename(i_repo_path)
+        repo_folder_name = os.path.basename(i_repo_path.rstrip('/'))
 
         repo_path = os.path.join(self.repos_path, repo_folder_name)