Merge pull request #192 from mlcommons/dev

Merge Dev

Author: anandhu-eng

.github/workflows/test-mlc-docker-core.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.12", "3.8"]
+        python-version: ["3.13", "3.8"]
         os: ["ubuntu-latest", "windows-latest", "macos-latest"]
         exclude:
           - os: windows-latest
@@ -46,12 +46,12 @@ jobs:
 
     - name: Test --docker_dt for running in detached mode
       run: |
-        mlc docker run --tags=detect,os --docker_dt
+        mlc docker-run script --tags=detect,os --docker_dt
         docker stop $(docker ps -aq)
 
     - name: Test --docker_detached for running in detached mode
       run: |
-        mlc docker run --tags=detect,os --docker_detached
+        mlcd detect,os --docker_detached
         docker stop $(docker ps -aq)
 
     - name: Test --docker_cache 
@@ -64,4 +64,4 @@ jobs:
 
     - name: Test --dockerfile_recreate
       run: |
-        mlc docker run --tags=detect,os --docker_dt --docker_cache=no --docker_rebuild --dockerfile_recreate
+        mlc docker-run script --tags=detect,os --docker_dt --docker_cache=no --docker_rebuild --dockerfile_recreate

README.md

@@ -57,7 +57,7 @@ Each target has its own set of specific actions to tailor automation workflows a
 
 | Target | Action          |
 |--------|-----------------|
-| script    | run, find/search, rm, mv, cp, add, test, docker, show       |
+| script    | run, find/search, rm, mv, cp, add, test, docker-run, show, experiment, doc   |
 | cache    | find/search, rm, show  |
 | repo    | pull, search, rm, list, find/search , add        |
 
@@ -115,9 +115,6 @@ classDiagram
         +show(args)
         +list(args)
     }
-    class CfgAction {
-        +load(args)
-    }
     class Index {
         +add(meta, folder_type, path, repo)
         +get_index(folder_type, uid)
@@ -148,8 +145,6 @@ classDiagram
     Action <|-- RepoAction
     Action <|-- ScriptAction
     Action <|-- CacheAction
-    Action <|-- ExperimentAction
-    Action <|-- CfgAction
     RepoAction o-- Repo
     ScriptAction o-- Automation
     CacheAction o-- Index

VERSION

@@ -1 +1 @@
-1.2.0
+1.1.9

mlc/action.py

@@ -42,6 +42,8 @@ def access(self, options):
             return {'return': 1, 'error': "'action' key is required in options"}
         #logger.info(f"options = {options}")
 
+        action_name = action_name.replace("-", "_")
+
         action_target = options.get('target')
         if not action_target:
             action_target = options.get('automation', 'script')  # Default to script if not provided

mlc/index.py

@@ -178,6 +178,6 @@ def _save_indices(self):
             try:
                 with open(output_file, "w") as f:
                     json.dump(index_data, f, indent=4, cls=CustomJSONEncoder)
-                logger.debug(f"Shared index for {folder_type} saved to {output_file}.")
+                #logger.debug(f"Shared index for {folder_type} saved to {output_file}.")
             except Exception as e:
                 logger.error(f"Error saving shared index for {folder_type}: {e}")

mlc/main.py

@@ -3,7 +3,7 @@
 import sys
 from pathlib import Path
 import inspect
-
+import shlex
 from . import utils
 
 from .action import Action, default_parent
@@ -74,9 +74,11 @@ def search(self, i):
         return {'return': 0, 'list': result}
         #indices
 
+mlc_run_cmd = None
 
 def mlc_expand_short(action, target = "script"):
-
+    global mlc_run_cmd
+    mlc_run_cmd = shlex.join(sys.argv)
     # Insert the positional argument into sys.argv for the main function
     sys.argv.insert(1, action)
     sys.argv.insert(2, target)
@@ -88,6 +90,10 @@ def mlcr():
     mlc_expand_short("run")
 def mlcd():
     mlc_expand_short("docker")
+def mlcdr():
+    mlc_expand_short("docker")
+def mlcrr():
+    mlc_expand_short("remote-run")
 def mlce():
     mlc_expand_short("experiment")
 def mlct():
@@ -115,7 +121,86 @@ def process_console_output(res, target, action, run_args):
 if default_parent is None:
     default_parent = Action()
 
-# Main CLI function
+
+log_flag_aliases = {'-v': '--verbose', '-s': '--silent'}
+log_levels = {'--verbose': logging.DEBUG, '--silent': logging.WARNING}
+
+
+def build_pre_parser():
+    pre_parser = argparse.ArgumentParser(add_help=False)
+    pre_parser.add_argument("action", nargs="?", help="Top-level action (run, build, help, etc.)")
+    pre_parser.add_argument("target", choices=['run', 'script', 'cache', 'repo', 'repos'], nargs="?", help="Target (repo, script, cache, ...)")
+    pre_parser.add_argument("-h", "--help", action="store_true")
+    return pre_parser
+
+
+def build_parser(pre_args):
+    parser = argparse.ArgumentParser(prog="mlc", description="Manage repos, scripts, and caches.", add_help=False)
+    subparsers = parser.add_subparsers(dest="command", required=not pre_args.help)
+
+    # General commands
+    for action in ['run', 'pull', 'test', 'add', 'show', 'list', 'find', 'search', 'rm', 'cp', 'mv', 'help']:
+        p = subparsers.add_parser(action, add_help=False)
+        p.add_argument('target', choices=['repo', 'repos', 'script', 'cache'])
+        p.add_argument('details', nargs='?', help='Details or identifier (optional)')
+        p.add_argument('extra', nargs=argparse.REMAINDER)
+
+    # Script-only
+    for action in ['docker', 'docker-run', 'experiment', 'remote-run', 'doc', 'lint']:
+        p = subparsers.add_parser(action, add_help=False)
+        p.add_argument('target', choices=['script', 'run'])
+        p.add_argument('details', nargs='?', help='Details or identifier (optional)')
+        p.add_argument('extra', nargs=argparse.REMAINDER)
+
+    # Load cfg
+    load_parser = subparsers.add_parser("load", add_help=False)
+    load_parser.add_argument("target", choices=["cfg"])
+    return parser
+
+
+def configure_logging(args):
+    if hasattr(args, 'extra') and args.extra:
+        args.extra[:] = [log_flag_aliases.get(a, a) for a in args.extra]
+        for flag, level in log_levels.items():
+            if flag in args.extra:
+                logger.setLevel(level)
+                args.extra.remove(flag)
+
+
+def build_run_args(args):
+    global mlc_run_cmd
+    res = utils.convert_args_to_dictionary(getattr(args, 'extra', []))
+    if res['return'] > 0:
+        return res
+
+    run_args = res['args_dict']
+    if not mlc_run_cmd:
+        mlc_run_cmd = shlex.join(sys.argv)
+    run_args['mlc_run_cmd'] = mlc_run_cmd
+
+    if args.command in ['pull', 'rm', 'add', 'find'] and args.target == "repo":
+        run_args['repo'] = args.details
+
+    if args.command in ['docker', 'docker-run', 'experiment', 'remote-run', 'doc', 'lint'] and args.target == "run":
+        #run_args['target'] = 'script' #dont modify this as script might have target as in input
+        args.target = "script"
+
+    if args.details and not utils.is_uid(args.details) and not run_args.get("tags") and args.target in ["script", "cache"]:
+        run_args['tags'] = args.details
+
+    if not run_args.get('details') and args.details:
+        run_args['details'] = args.details
+
+    if args.command in ["cp", "mv"]:
+        run_args['target'] = args.target
+        if args.details:
+            run_args['src'] = args.details
+        if args.extra:
+            run_args['dest'] = args.extra[0]
+
+    return run_args
+
+
 def main():
     """
     MLCFlow is a CLI tool for managing repos, scripts, and caches.
@@ -130,11 +215,11 @@ def main():
 
     Each target has a specific set of actions to tailor automation workflows, as shown below:
     
-    | Target  | Actions                                               |
-    |---------|-------------------------------------------------------|
-    | script  | run, find/search, rm, mv, cp, add, test, docker, show |
-    | cache   | find/search, rm, show                                 |
-    | repo    | pull, search, rm, list, find/search                   |
+    | Target  | Actions                                                   |
+    |---------|-----------------------------------------------------------|
+    | script  | run, find/search, rm, mv, cp, add, test, docker-run, show |
+    | cache   | find/search, rm, show                                     |
+    | repo    | pull, search, rm, list, find/search                       |
     
     Example:
       mlc run script detect-os
@@ -154,18 +239,22 @@ def main():
       mlc run script --help
       mlc pull repo -h
     """
-
-    # First level parser for showing help
-    pre_parser = argparse.ArgumentParser(add_help=False)
-    pre_parser.add_argument("action", nargs="?", help="Top-level action (run, build, help, etc.)")
-    pre_parser.add_argument("target", choices=['run', 'script', 'cache', 'repo'], nargs="?", help="Potential target (repo, script, cache, ...)")
-    pre_parser.add_argument("-h", "--help", action="store_true")
+    pre_parser = build_pre_parser()
     pre_args, remaining_args = pre_parser.parse_known_args()
 
-    if pre_args.help and not any("--tags" in arg for arg in remaining_args):
+    parser = build_parser(pre_args)
+    args = parser.parse_args() if remaining_args or pre_args.target else pre_args
+    
+    if hasattr(args, 'command') and args.command:
+        args.command = args.command.replace("-", "_")
+
+    configure_logging(args)
+    run_args = build_run_args(args) if hasattr(args, "command") else {}
+
+    if pre_args.help and not "tags" in run_args:
         help_text = ""
         if pre_args.target == "run":
-            if pre_args.action == "docker":
+            if pre_args.action.startswith("docker"):
                 pre_args.target = "script"
             else:
                 logger.error(f"Invalid action-target {pre_args.action} - {pre_args.target} combination")
@@ -196,104 +285,26 @@ def main():
         if help_text != "":
             print(help_text)
         sys.exit(0)
-    
-    # parser for execution of the automation scripts
-    parser = argparse.ArgumentParser(prog='mlc', description='A CLI tool for managing repos, scripts, and caches.', add_help=False)
-
-    # Subparsers are added to main parser, allowing for different commands (subcommands) to be defined. 
-    # The chosen subcommand will be stored in the "command" attribute of the parsed arguments.
-    subparsers = parser.add_subparsers(dest='command', required=True)
-
-    # Script and Cache-specific subcommands
-    for action in ['run', 'pull', 'test', 'add', 'show', 'list', 'find', 'search', 'rm', 'cp', 'mv']:
-        action_parser = subparsers.add_parser(action, add_help=False)
-        action_parser.add_argument('target', choices=['repo', 'script', 'cache'], help='Target type (repo, script, cache).')
-        # the argument given after target and before any extra options like --tags will be stored in "details"
-        action_parser.add_argument('details', nargs='?', help='Details or identifier (optional for list).')
-        action_parser.add_argument('extra', nargs=argparse.REMAINDER, help='Extra options (e.g.,  -v)')
-
-    # Script specific subcommands
-    for action in ['docker', 'experiment', 'doc', 'lint']:
-        action_parser = subparsers.add_parser(action, add_help=False)
-        action_parser.add_argument('target', choices=['script', 'run'], help='Target type (script).')
-        # the argument given after target and before any extra options like --tags will be stored in "details"
-        action_parser.add_argument('details', nargs='?', help='Details or identifier (optional for list).')
-        action_parser.add_argument('extra', nargs=argparse.REMAINDER, help='Extra options (e.g.,  -v)')
-
-    for action in ['load']:
-        load_parser = subparsers.add_parser(action, add_help=False)
-        load_parser.add_argument('target', choices=['cfg'], help='Target type (cfg).')
-    
-    # Parse arguments
-    args = parser.parse_args()
-
-    #logger.info(f"Args = {args}")
-
-    # set log level for MLCFlow if -v/--verbose or -s/--silent is specified
-    log_flag_aliases = {
-        '-v': '--verbose',
-        '-s': '--silent'
-        }
-    
-    log_levels = {
-        '--verbose': logging.DEBUG,
-        '--silent': logging.WARNING
-        }
-
-    # Modify args.extra in place
-    args.extra[:] = [log_flag_aliases.get(arg, arg) for arg in args.extra]
-
-    # Set log level based on the first matching flag
-    for flag, level in log_levels.items():
-        if flag in args.extra:
-            logger.setLevel(level)
-            args.extra.remove(flag)
 
-    res = utils.convert_args_to_dictionary(args.extra)
-    if res['return'] > 0:
-        return res
-    
-    run_args = res['args_dict']
-
-    run_args['mlc_run_cmd'] = " ".join(sys.argv)
-    
-    if hasattr(args, 'repo') and args.repo:
-        run_args['repo'] = args.repo
+    # show repos alias list repo
+    if args.command in ("show"):
+        args.command = "list"
+    if args.target == "repos":
+        args.target = "repo"
 
-    if args.command in ['pull', 'rm', 'add', 'find']:
-        if args.target == "repo":
-            run_args['repo'] = args.details
-  
-    if args.command in ['docker', 'experiment', 'doc', 'lint']:
-        if args.target == "run":
-            run_args['target'] = 'script' #allowing run to be used for docker run instead of docker script
-            args.target = "script"
-
-    if hasattr(args, 'details') and args.details and not utils.is_uid(args.details) and not run_args.get("tags") and args.target in ["script", "cache"]:
-        run_args['tags'] = args.details
+    action = get_action(args.target, default_parent)
 
-    if not run_args.get('details') and args.details:
-        run_args['details'] = args.details
+    if not action or not hasattr(action, args.command):
+        logging.error("Error: '%s' is not supported for %s.", args.command, args.target)
+        sys.exit(1)
 
-    if args.command in ["cp", "mv"]:
-        run_args['target'] = args.target
-        if hasattr(args, 'details') and args.details:
-            run_args['src'] = args.details
-        if hasattr(args, 'extra') and args.extra:
-            run_args['dest'] = args.extra[0]
+    method = getattr(action, args.command)
+    res = method(run_args)
+    if res['return'] > 0:
+        logging.error(res.get('error', f"Error in {action}"))
+        raise Exception(f"An error occurred {res}")
 
-    # Get the action handler for other commands
-    action = get_action(args.target, default_parent)
-    # Dynamically call the method (e.g., run, list, show)
-    if action and hasattr(action, args.command):
-        method = getattr(action, args.command)
-        res = method(run_args)
-        if res['return'] > 0:
-            logger.error(res.get('error', f"Error in {action}"))
-            raise Exception(f"""An error occurred {res}""")
-        process_console_output(res, args.target, args.command, run_args)
-    else:
-        logger.error(f"Error: '{args.command}' is not supported for {args.target}.")
+    process_console_output(res, args.target, args.command, run_args)
 
 if __name__ == '__main__':
     main()