diff --git a/.github/workflows/test-mlc-core-actions.yaml b/.github/workflows/test-mlc-core-actions.yaml index 6a22b3cc8..d2cbed22a 100644 --- a/.github/workflows/test-mlc-core-actions.yaml +++ b/.github/workflows/test-mlc-core-actions.yaml @@ -256,3 +256,68 @@ jobs: run: | cd .github/scripts && python -c "import test_mlc_access as test; test.run_tests()" + test_mlc_help: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.8"] + os: ["ubuntu-latest", "windows-latest", "macos-latest"] + exclude: + - os: windows-latest + - os: macos-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Configure git longpaths (Windows) + if: matrix.os == 'windows-latest' + run: | + git config --system core.longpaths true + + - name: Install mlcflow from the pull request's source repository and branch + run: | + python -m pip install --upgrade pip + python -m pip install --ignore-installed --verbose pip setuptools + python -m pip install . + - name: MLC intro + run: | + mlc help + - name: MLC scripts - full + run: | + mlc help script + - name: MLC scripts - individual actions + run: | + mlc help add script + mlc help docker script + mlc help find script + mlc help search script + mlc help list script + mlc help rm script + mlc help run script + mlc help show script + mlc help test script + - name: MLC repos - full + run: | + mlc help repo + - name: MLC repos - individual actions + run: | + mlc help add repo + mlc help find repo + mlc help pull repo + mlc help list repo + mlc help rm repo + - name: MLC cache - full + run: | + mlc help cache + - name: MLC cache - individual actions + run: | + mlc help find cache + mlc help list cache + mlc help rm cache + mlc help search cache + mlc help show cache diff --git a/docs/targets/script/index.md b/docs/targets/script/index.md index 12867692b..715a72bdc 100644 --- a/docs/targets/script/index.md +++ b/docs/targets/script/index.md @@ -18,9 +18,9 @@ MLC scripts can be identified in different ways: --- -## **Find** +## **Find(Alias: Search)** -The `find` action retrieves the path of scripts available in MLC repositories. +The `find/search` action retrieves the path of scripts available in MLC repositories. ### **Syntax Variations** @@ -135,6 +135,17 @@ mlc rm script --tags=detect,os -f --- +## **List** + +Lists all the scripts and their paths present in repos which are registered in MLC. + +**Example Command:** +```bash +mlc list script +``` + +--- + ## **Run** Executes a script from an MLC repository. @@ -171,8 +182,11 @@ mlc run script --tags=detect,os -j - *``: In addition to the above options an `mlcr` command also takes any input specified with in a script meta in `input_mappings` as its input. **Log Levels** + - *Default* : Provides all logs at the `info` level. + - *Silent* `[--silent/-s]` : Disables all `debug` and `info` level logs from automation. Logs from individual scripts will still be visible. + - *verbose* `[--verbose/-v]` : Displays both `info` and `debug` level logs from automation, along with logs from individual scripts.
diff --git a/mlc/cache_action.py b/mlc/cache_action.py index 609702cf7..30c7b97e0 100644 --- a/mlc/cache_action.py +++ b/mlc/cache_action.py @@ -5,6 +5,17 @@ from .logger import logger class CacheAction(Action): + """ + #################################################################################################################### + Cache Action + #################################################################################################################### + Currently, the following actions are supported for Cache: + 1. find/search + 2. show + 3. list + 4. remove(rm) + + """ def __init__(self, parent=None): #super().__init__(parent) @@ -12,6 +23,23 @@ def __init__(self, parent=None): self.__dict__.update(vars(parent)) def search(self, i): + """ + #################################################################################################################### + Target: Cache + Action: Find (Alias: Search) + #################################################################################################################### + + The `find` (or `search`) action retrieves the path of cache generated by a particular script. + + Syntax: + + mlc find cache --tags= + + Example Command: + + mlc find cache --tags=get,dataset,igbh + + """ i['target_name'] = "cache" #logger.debug(f"Searching for cache with input: {i}") return self.parent.search(i) @@ -19,11 +47,55 @@ def search(self, i): find = search def rm(self, i): + """ + #################################################################################################################### + Target: Cache + Action: Remove(rm) + #################################################################################################################### + + The `rm` action removes one or more caches generated while running scripts through MLCFlow. + + Syntax: + + mlc rm cache --tags= + + Options: + 1. `-f`: Force removes caches without confirmation. Without `-f`, the user will be prompted for confirmation before deletion. + + To remove all generated caches, use: + + mlc rm cache + + Example Command: + + mlc rm cache --tags=get,dataset,igbh + + """ i['target_name'] = "cache" #logger.debug(f"Removing cache with input: {i}") return self.parent.rm(i) def show(self, run_args): + """ + #################################################################################################################### + Target: Cache + Action: Show + #################################################################################################################### + + Retrieves the path and metadata of caches generated while running scripts through MLCFlow. + + Syntax: + + mlc show cache --tags= + + Example Command: + + mlc show cache --tags=get,dataset,igbh + + Note: + - The `find` action is a subset of `show`. It retrieves only the path of the searched cache. + + """ self.action_type = "cache" res = self.search(run_args) logger.info(f"Showing cache with tags: {run_args.get('tags')}") @@ -59,6 +131,19 @@ def show(self, run_args): return {'return': 0} def list(self, args): + """ + #################################################################################################################### + Target: Cache + Action: List + #################################################################################################################### + + Lists all cached items along with their paths. + + Example Command: + + mlc list cache + + """ self.action_type = "cache" run_args = {"fetch_all": True} # to fetch the details of all the caches generated diff --git a/mlc/main.py b/mlc/main.py index 95e2af53d..f18cecc68 100644 --- a/mlc/main.py +++ b/mlc/main.py @@ -2,6 +2,7 @@ import os import sys from pathlib import Path +import inspect from . import utils @@ -17,7 +18,6 @@ from .logger import logger - class Automation: action_object = None automation_type = None @@ -103,7 +103,37 @@ def process_console_output(res, target, action, run_args): # Main CLI function def main(): - parser = argparse.ArgumentParser(prog='mlc', description='A CLI tool for managing repos, scripts, and caches.') + """ + MLCFlow is a CLI tool for managing repos, scripts, and caches. + This framework is designed to streamline automation workflows for MLPerf benchmarks more efficiently. + You can also use this tool for any of your workflow automation tasks. + + MLCFlow CLI operates using actions and targets. It enables users to perform actions on specified targets using the following syntax: + + mlc [options] + + Here, actions represent the operations to be performed, and the target is the object on which the action is executed. + + Each target has a specific set of actions to tailor automation workflows, as shown below: + + | Target | Actions | + |---------|-------------------------------------------------------| + | script | run, find/search, rm, mv, cp, add, test, docker, show | + | cache | find/search, rm, show | + | repo | pull, search, rm, list, find/search | + + Example: + mlc run script detect-os + + For help related to a particular target, run: + + mlc help + + For help related to a specific action for a target, run: + + mlc help + """ + parser = argparse.ArgumentParser(prog='mlc', description='A CLI tool for managing repos, scripts, and caches.', add_help=False) # Subparsers are added to main parser, allowing for different commands (subcommands) to be defined. # The chosen subcommand will be stored in the "command" attribute of the parsed arguments. @@ -118,7 +148,7 @@ def main(): action_parser.add_argument('extra', nargs=argparse.REMAINDER, help='Extra options (e.g., -v)') # Script and specific subcommands - for action in ['docker', 'help']: + for action in ['docker']: action_parser = subparsers.add_parser(action, help=f'{action.capitalize()} a target.') action_parser.add_argument('target', choices=['script', 'run'], help='Target type (script).') # the argument given after target and before any extra options like --tags will be stored in "details" @@ -129,10 +159,49 @@ def main(): load_parser = subparsers.add_parser(action, help=f'{action.capitalize()} a target.') load_parser.add_argument('target', choices=['cfg'], help='Target type (cfg).') + for action in ['help']: + action_parser = subparsers.add_parser(action, help=f'{action.capitalize()} a target.') + action_parser.add_argument('action', help='action type (run).', nargs='?', default=None) + action_parser.add_argument('target', choices=['script', 'cache', 'repo'], help='Target type (script).', nargs='?', default=None) + # Parse arguments args = parser.parse_args() + # handle help in mlcflow + if args.command in ['help']: + help_text = "" + if not args.action and not args.target: + print(main.__doc__) + sys.exit(0) + elif args.action and not args.target: + if args.action not in ['script', 'cache', 'repo']: + logger.error(f"Invalid target {args.action}") + raise Exception(f"""Invalid target {args.action}""") + else: + args.target = args.action + args.action = None + actions = get_action(args.target, default_parent) + help_text += actions.__doc__ + + # iterate through every method + for method_name, method in inspect.getmembers(actions.__class__, inspect.isfunction): + method = getattr(actions, method_name) + if method.__doc__ and not method.__doc__.startswith("_"): + help_text += method.__doc__ + print(help_text) + sys.exit(0) + else: + actions = get_action(args.target, default_parent) + try: + method = getattr(actions, args.action) + help_text += actions.__doc__ + help_text += method.__doc__ + print(help_text) + except: + logger.error(f"Error: '{args.action}' is not supported for {args.target}.") + sys.exit(0) + #logger.info(f"Args = {args}") res = utils.convert_args_to_dictionary(args.extra) diff --git a/mlc/repo_action.py b/mlc/repo_action.py index 491fa334d..df30d7764 100644 --- a/mlc/repo_action.py +++ b/mlc/repo_action.py @@ -9,6 +9,31 @@ from .logger import logger class RepoAction(Action): + """ + #################################################################################################################### + Repo Action + #################################################################################################################### + + Currently, the following actions are supported for Repos: + 1. add + 2. find + 3. pull + 4. list + 5. remove(rm) + + Repositories in MLCFlow can be identified using any of the following methods: + + Using MLC repo folder name format: (e.g.,mlcommons@mlperf-automations) + Using alias: (e.g., mlcommons@mlperf-automations) + Using UID: (e.g., 9cf241afa6074c89) + Using both alias and UID: , (e.g., mlcommons@mlperf-automations,9cf241afa6074c89) + Using URL: (e.g., https://github.com/mlcommons/mlperf-automations) + + Note: + + - repo uid and repo alias for a particular MLC repository can be found inside the meta.yml file. + + """ def __init__(self, parent=None): #super().__init__(parent) @@ -17,6 +42,31 @@ def __init__(self, parent=None): def add(self, run_args): + """ + #################################################################################################################### + Target: Repo + Action: Add + #################################################################################################################### + + The `add` action is used to create a new MLC repository and register it in MLCFlow. + The newly created repo folder will be stored inside the `repos` folder within the parent MLC directory. + + Example Command: + + mlc add repo mlcommons@script-automations + + Example Output: + + anandhu@anandhu-VivoBook-ASUSLaptop-X515UA-M515UA:~$ mlc add repo mlcommons@script-automations + [2025-02-19 16:34:37,570 main.py:1085 INFO] - New repo path: /home/anandhu/MLC/repos/mlcommons@script-automations + [2025-02-19 16:34:37,573 main.py:1126 INFO] - Added new repo path: /home/anandhu/MLC/repos/mlcommons@script-automations + [2025-02-19 16:34:37,573 main.py:1130 INFO] - Updated repos.json at /home/anandhu/MLC/repos/repos.json + + Note: + - repo_uid is not supported in the add action for repo target, as the UID is assigned automatically when the repository + is created. + + """ if not run_args['repo']: logger.error("The repository to be added is not specified") return {"return": 1, "error": "The repository to be added is not specified"} @@ -99,77 +149,95 @@ def unregister_repo(self, repo_path): def find(self, run_args): - # Get repos_list using the existing method - repos_list = self.load_repos_and_meta() - if(run_args.get('item', run_args.get('artifact'))): - repo = run_args.get('item', run_args.get('artifact')) - else: - repo = run_args.get('repo', run_args.get('item', run_args.get('artifact'))) + """ + #################################################################################################################### + Target: Repo + Action: Find + #################################################################################################################### + + find action retrieves the path of a specific repository registered in MLCFlow. + + Example Command: + + mlc find repo mlcommons@script-automations + + Example Output: + + anandhu@anandhu-VivoBook-ASUSLaptop-X515UA-M515UA:~$ mlc find repo mlcommons@mlperf-automations + [2025-02-19 15:32:18,352 main.py:1737 INFO] - Item path: /home/anandhu/MLC/repos/mlcommons@mlperf-automations - # Check if repo is None or empty - if not repo: - return {"return": 1, "error": "Please enter a Repo Alias, Repo UID, or Repo URL in one of the following formats:\n" + """ + # Get repos_list using the existing method + repos_list = self.load_repos_and_meta() + if(run_args.get('item', run_args.get('artifact'))): + repo = run_args.get('item', run_args.get('artifact')) + else: + repo = run_args.get('repo', run_args.get('item', run_args.get('artifact'))) + + # Check if repo is None or empty + if not repo: + return {"return": 1, "error": "Please enter a Repo Alias, Repo UID, or Repo URL in one of the following formats:\n" "- @\n" "- \n" "- \n" "- \n" "- ,"} - # Handle the different repo input formats - repo_name = None - repo_uid = None - - # Check if the repo is in the format of a repo UID (alphanumeric string) - if self.is_uid(repo): - repo_uid = repo - if "," in repo: - repo_split = repo.split(",") - repo_name = repo_split[0] - if len(repo_split) > 1: - repo_uid = repo_split[1] - elif "@" in repo: - repo_name = repo - elif "github.com" in repo: - result = self.github_url_to_user_repo_format(repo) - if result["return"] == 0: - repo_name = result["value"] - else: - return result - - # Check if repo_name exists in repos.json - matched_repo_path = None - for repo_obj in repos_list: - if repo_name and repo_name == os.path.basename(repo_obj.path) : - matched_repo_path = repo_obj - break - - # Search through self.repos for matching repos - lst = [] - for i in self.repos: - if repo_uid and i.meta['uid'] == repo_uid: - lst.append(i) - elif repo_name == i.meta['alias']: - lst.append(i) - elif self.is_uid(repo) and not any(i.meta['uid'] == repo_uid for i in self.repos): - return {"return": 1, "error": f"No repository with UID: '{repo_uid}' was found"} - elif "," in repo and not matched_repo_path and not any(i.meta['uid'] == repo_uid for i in self.repos) and not any(i.meta['alias'] == repo_name for i in self.repos): - return {"return": 1, "error": f"No repository with alias: '{repo_name}' and UID: '{repo_uid}' was found"} - elif not matched_repo_path and not any(i.meta['alias'] == repo_name for i in self.repos) and not any(i.meta['uid'] == repo_uid for i in self.repos ): - return {"return": 1, "error": f"No repository with alias: '{repo_name}' was found"} + # Handle the different repo input formats + repo_name = None + repo_uid = None + + # Check if the repo is in the format of a repo UID (alphanumeric string) + if self.is_uid(repo): + repo_uid = repo + if "," in repo: + repo_split = repo.split(",") + repo_name = repo_split[0] + if len(repo_split) > 1: + repo_uid = repo_split[1] + elif "@" in repo: + repo_name = repo + elif "github.com" in repo: + result = self.github_url_to_user_repo_format(repo) + if result["return"] == 0: + repo_name = result["value"] + else: + return result + + # Check if repo_name exists in repos.json + matched_repo_path = None + for repo_obj in repos_list: + if repo_name and repo_name == os.path.basename(repo_obj.path) : + matched_repo_path = repo_obj + break + + # Search through self.repos for matching repos + lst = [] + for i in self.repos: + if repo_uid and i.meta['uid'] == repo_uid: + lst.append(i) + elif repo_name == i.meta['alias']: + lst.append(i) + elif self.is_uid(repo) and not any(i.meta['uid'] == repo_uid for i in self.repos): + return {"return": 1, "error": f"No repository with UID: '{repo_uid}' was found"} + elif "," in repo and not matched_repo_path and not any(i.meta['uid'] == repo_uid for i in self.repos) and not any(i.meta['alias'] == repo_name for i in self.repos): + return {"return": 1, "error": f"No repository with alias: '{repo_name}' and UID: '{repo_uid}' was found"} + elif not matched_repo_path and not any(i.meta['alias'] == repo_name for i in self.repos) and not any(i.meta['uid'] == repo_uid for i in self.repos ): + return {"return": 1, "error": f"No repository with alias: '{repo_name}' was found"} - # Append the matched repo path - if(len(lst)==0): - lst.append(matched_repo_path) + # Append the matched repo path + if(len(lst)==0): + lst.append(matched_repo_path) - return {'return': 0, 'list': lst} + return {'return': 0, 'list': lst} def github_url_to_user_repo_format(self, url): - """ - Converts a GitHub repo URL to user@repo_name format. + # """ + # Converts a GitHub repo URL to user@repo_name format. - :param url: str, GitHub repository URL (e.g., https://github.com/user/repo_name.git) - :return: str, formatted as user@repo_name - """ + # :param url: str, GitHub repository URL (e.g., https://github.com/user/repo_name.git) + # :return: str, formatted as user@repo_name + # """ # Regex to match GitHub URLs pattern = r"(?:https?://)?(?:www\.)?github\.com/([^/]+)/([^/.]+)(?:\.git)?" @@ -293,6 +361,50 @@ def pull_repo(self, repo_url, branch=None, checkout = None, tag = None, pat = No return {'return': 1, 'error': f"Error pulling repository: {str(e)}"} def pull(self, run_args): + """ + #################################################################################################################### + Target: Repo + Action: Pull + #################################################################################################################### + + The `pull` action clones an MLC repository and registers it in MLC. + + If the repository already exists locally in the MLC repos directory, it fetches the latest changes only if there are no + uncommited modifications(excluding untracked files/folders). The `pull` action could be also used to checkout + to a particular branch, commit or release tag using flags --checkout and --tag. + + Example Command: + + mlc pull repo mlcommons@script-automations + + + - `--checkout `: Checks out a specific commit after cloning (applicable when the repository exists locally). + - `--branch `: Checks out a specific branch **while cloning** a new repository. + - `--tag `: Checks out a particular release tag. + - `--pat ` or `--ssh`: Clones a private repository using a personal access token or SSH. + + Example Output: + + anandhu@anandhu-VivoBook-ASUSLaptop-X515UA-M515UA:~$ mlc pull repo mlcommons@mlperf-automations + [2025-02-19 16:46:27,208 main.py:1260 INFO] - Cloning repository https://github.com/mlcommons/mlperf-automations.git + to /home/anandhu/MLC/repos/mlcommons@mlperf-automations... + Cloning into '/home/anandhu/MLC/repos/mlcommons@mlperf-automations'... + remote: Enumerating objects: 77610, done. + remote: Counting objects: 100% (2199/2199), done. + remote: Compressing objects: 100% (1103/1103), done. + remote: Total 77610 (delta 1616), reused 1109 (delta 1095), pack-reused 75411 (from 2) + Receiving objects: 100% (77610/77610), 18.36 MiB | 672.00 KiB/s, done. + Resolving deltas: 100% (53818/53818), done. + [2025-02-19 16:46:57,604 main.py:1288 INFO] - Repository successfully pulled. + [2025-02-19 16:46:57,605 main.py:1289 INFO] - Registering the repo in repos.json + [2025-02-19 16:46:57,605 main.py:1126 INFO] - Added new repo path: /home/anandhu/MLC/repos/mlcommons@mlperf-automations + [2025-02-19 16:46:57,606 main.py:1130 INFO] - Updated repos.json at /home/anandhu/MLC/repos/repos.json + + Note: + - repo_uid and repo_alias are not supported in the pull action for the repo target. + - Only one of --checkout, --branch, or --tag should be specified at a time. + + """ repo_url = run_args.get('repo', run_args.get('url', 'repo')) if not repo_url or repo_url == "repo": for repo_object in self.repos: @@ -320,6 +432,34 @@ def pull(self, run_args): def list(self, run_args): + """ + #################################################################################################################### + Target: Repo + Action: List + #################################################################################################################### + + The `list` action displays all registered MLC repositories along with their aliases and paths. + + Example Command: + + mlc list repo + + Example Output: + + anandhu@anandhu-VivoBook-ASUSLaptop-X515UA-M515UA:~$ mlc list repo + [2025-02-19 16:56:31,847 main.py:1349 INFO] - Listing all repositories. + + Repositories: + ------------- + - Alias: local + Path: /home/anandhu/MLC/repos/local + + - Alias: mlcommons@mlperf-automations + Path: /home/anandhu/MLC/repos/mlcommons@mlperf-automations + ------------- + [2025-02-19 16:56:31,850 main.py:1356 INFO] - Repository listing ended + + """ logger.info("Listing all repositories.") print("\nRepositories:") print("-------------") @@ -331,7 +471,31 @@ def list(self, run_args): return {"return": 0} def rm(self, run_args): + """ + #################################################################################################################### + Target: Repo + Action: rm + #################################################################################################################### + + The `rm` action removes a specified repository from MLCFlow, deleting both the repo folder and its registration. + If there are any modified local changes, the user will be prompted for confirmation unless the `-f` flag is used + for force removal. + + Example Command: + + mlc rm repo mlcommons@mlperf-automations + Example Output: + + anandhu@anandhu-VivoBook-ASUSLaptop-X515UA-M515UA:~$ mlc rm repo mlcommons@mlperf-automations + [2025-02-19 17:01:59,483 main.py:1360 INFO] - rm command has been called for repo. This would delete the repo folder and unregister the repo from repos.json + [2025-02-19 17:01:59,521 main.py:1380 INFO] - No local changes detected. Removing repo... + [2025-02-19 17:01:59,581 main.py:1384 INFO] - Repo mlcommons@mlperf-automations residing in path /home/anandhu/MLC/repos/mlcommons@mlperf-automations has been successfully removed + [2025-02-19 17:01:59,581 main.py:1385 INFO] - Checking whether the repo was registered in repos.json + [2025-02-19 17:01:59,581 main.py:1134 INFO] - Unregistering the repo in path /home/anandhu/MLC/repos/mlcommons@mlperf-automations + [2025-02-19 17:01:59,581 main.py:1144 INFO] - Path: /home/anandhu/MLC/repos/mlcommons@mlperf-automations has been removed. + + """ if not run_args['repo']: logger.error("The repository to be removed is not specified") return {"return": 1, "error": "The repository to be removed is not specified"} diff --git a/mlc/script_action.py b/mlc/script_action.py index 3bc999382..96a45c266 100644 --- a/mlc/script_action.py +++ b/mlc/script_action.py @@ -6,12 +6,49 @@ from .logger import logger class ScriptAction(Action): + """ + #################################################################################################################### + Script Action + #################################################################################################################### + + The following actions are currently supported for scripts: + 1. add + 2. find + 3. show + 3. Move(mv) + 5. Remove(rm) + 6. Copy(cp) + 7. Run + 8. Docker + 9. Test + + Scripts in MLCFlow can be identified using different methods: + + Using tags: --tags= (e.g., --tags=detect,os) + Using alias: (e.g., detect-os) + Using UID: (e.g., 5b4e0237da074764) + Using both alias and UID: , (e.g., detect-os,5b4e0237da074764) + + """ parent = None def __init__(self, parent=None): self.parent = parent self.__dict__.update(vars(parent)) def search(self, i): + """ + #################################################################################################################### + Target: Script + Action: Find (Alias: Search) + #################################################################################################################### + + The `find` (or `search`) action retrieves the path of scripts available in MLC repositories. + + Example Command: + + mlc find script --tags=detect,os -f + + """ if not i.get('target_name'): i['target_name'] = "script" res = self.parent.search(i) @@ -21,12 +58,56 @@ def search(self, i): find = search def rm(self, i): + """ + #################################################################################################################### + Target: Script + Action: Remove(rm) + #################################################################################################################### + + The `remove` (`rm`) action deletes one or more scripts from MLC repositories. + + Example Command: + + mlc rm script --tags=detect,os -f + + """ if not i.get('target_name'): i['target_name'] = "script" logger.debug(f"Removing script with input: {i}") return self.parent.rm(i) def show(self, run_args): + """ + #################################################################################################################### + Target: Script + Action: Show + #################################################################################################################### + + The `show` action retrieves the path and metadata of the searched script in MLC repositories. + + Example Command: + + mlc show script --tags=detect,os + + Example Output: + + arjun@intel-spr-i9:~$ mlc show script --tags=detect,os + [2025-02-14 02:56:16,604 main.py:1404 INFO] - Showing script with tags: detect,os + Location: /home/arjun/MLC/repos/gateoverflow@mlperf-automations/script/detect-os: + Main Script Meta: + uid: 863735b7db8c44fc + alias: detect-os + tags: ['detect-os', 'detect', 'os', 'info'] + new_env_keys: ['MLC_HOST_OS_*', '+MLC_HOST_OS_*', 'MLC_HOST_PLATFORM_*', 'MLC_HOST_PYTHON_*', 'MLC_HOST_SYSTEM_NAME', + 'MLC_RUN_STATE_DOCKER', '+PATH'] + new_state_keys: ['os_uname_*'] + ...................................................... + For full script meta, see meta file at /home/arjun/MLC/repos/gateoverflow@mlperf-automations/script/detect-os/meta.yaml + + Note: + - The `find` action is a subset of `show`, retrieving only the path of the searched script in MLC repositories. + + """ self.action_type = "script" res = self.search(run_args) if res['return'] > 0: @@ -50,18 +131,47 @@ def show(self, run_args): def add(self, i): """ - Adds a new script to the repository. + #################################################################################################################### + Target: Script + Action: Add + #################################################################################################################### + + The `add` action creates a new script in a registered MLC repository. + + Syntax: + + mlc add script :new_script --tags=benchmark - Args: - i (dict): Input dictionary with the following keys: - - item_repo (tuple): Repository alias and UID (default: local repo). - - item (str): Item alias and optional UID in "alias,uid" format. - - tags (str): Comma-separated tags. - - yaml (bool): Whether to save metadata in YAML format. Defaults to JSON. + Options: + --template_tags: A comma-separated list of tags to create a new MLC script based on existing templates. + + Example Output: + + arjun@intel-spr-i9:~$ mlc add script gateoverflow@mlperf-automations --tags=benchmark --template_tags=app,mlperf,inference + More than one script found for None: + 1. /home/arjun/MLC/repos/gateoverflow@mlperf-automations/script/app-mlperf-inference-mlcommons-python + 2. /home/arjun/MLC/repos/gateoverflow@mlperf-automations/script/app-mlperf-inference-ctuning-cpp-tflite + 3. /home/arjun/MLC/repos/gateoverflow@mlperf-automations/script/app-mlperf-inference + 4. /home/arjun/MLC/repos/gateoverflow@mlperf-automations/script/app-mlperf-inference-mlcommons-cpp + Select the correct one (enter number, default=1): 1 + [2025-02-14 02:58:33,453 main.py:664 INFO] - Folder successfully copied from /home/arjun/MLC/repos/ + gateoverflow@mlperf-automations/script/app-mlperf-inference-mlcommons-python to /home/arjun/MLC/repos/ + gateoverflow@mlperf-automations/script/gateoverflow@mlperf-automations - Returns: - dict: Result of the operation with 'return' code and error/message if applicable. """ + # """ + # Adds a new script to the repository. + + # Args: + # i (dict): Input dictionary with the following keys: + # - item_repo (tuple): Repository alias and UID (default: local repo). + # - item (str): Item alias and optional UID in "alias,uid" format. + # - tags (str): Comma-separated tags. + # - yaml (bool): Whether to save metadata in YAML format. Defaults to JSON. + + # Returns: + # dict: Result of the operation with 'return' code and error/message if applicable. + # """ # Determine repository if i.get('details'): item = i['details'] @@ -135,16 +245,96 @@ def call_script_module_function(self, function_name, run_args): return {'return': 1, 'error': 'ScriptAutomation class not found in the script.'} def docker(self, run_args): + """ + #################################################################################################################### + Target: Script + Action: Docker + #################################################################################################################### + + The `docker` action runs scripts inside a containerized environment. + + An MLCFlow script can be executed inside a Docker container using either of the following syntaxes: + + 1. Docker Run: mlc docker run --tags=