wip: using threads to build command table

Author: DanielMicrosoft

.vscode/launch.json

@@ -23,7 +23,7 @@
             "program": "${workspaceRoot}/src/azure-cli/azure/cli/__main__.py",
             "cwd": "${workspaceRoot}",
             "args": [
-                "--version"
+                "Version"
             ],
             "console": "externalTerminal",
         },

src/azure-cli-core/azure/cli/core/__init__.py

@@ -9,6 +9,7 @@
 import os
 import sys
 import timeit
+from concurrent.futures import ThreadPoolExecutor
 
 from knack.cli import CLI
 from knack.commands import CLICommandsLoader
@@ -258,31 +259,48 @@ def _update_command_table_from_modules(args, command_modules=None):
             logger.debug("Loading command modules:")
             logger.debug(self.header_mod)
 
-            for mod in [m for m in command_modules if m not in BLOCKED_MODS]:
+            print(f"*** Starting SUT***")
+            start_time_wrapper = timeit.default_timer()
+            
+            def load_module_threaded(mod):
                 try:
                     start_time = timeit.default_timer()
                     module_command_table, module_group_table = _load_module_command_loader(self, args, mod)
                     import_module_breaking_changes(mod)
-                    for cmd in module_command_table.values():
-                        cmd.command_source = mod
-                    self.command_table.update(module_command_table)
-                    self.command_group_table.update(module_group_table)
-
                     elapsed_time = timeit.default_timer() - start_time
-                    logger.debug(self.item_format_string, mod, elapsed_time,
-                                 len(module_group_table), len(module_command_table))
-                    count += 1
-                    cumulative_elapsed_time += elapsed_time
-                    cumulative_group_count += len(module_group_table)
-                    cumulative_command_count += len(module_command_table)
+                    return (mod, module_command_table, module_group_table, elapsed_time, None)
                 except Exception as ex:  # pylint: disable=broad-except
-                    # Changing this error message requires updating CI script that checks for failed
-                    # module loading.
-                    from azure.cli.core import telemetry
-                    logger.error("Error loading command module '%s': %s", mod, ex)
-                    telemetry.set_exception(exception=ex, fault_type='module-load-error-' + mod,
-                                            summary='Error loading module: {}'.format(mod))
-                    logger.debug(traceback.format_exc())
+                    return (mod, {}, {}, 0, ex)
+            
+            # Use ThreadPoolExecutor to load modules in parallel
+            with ThreadPoolExecutor(max_workers=4) as executor:
+                futures = [executor.submit(load_module_threaded, mod) 
+                          for mod in command_modules if mod not in BLOCKED_MODS]
+                
+                for future in futures:
+                    mod, module_command_table, module_group_table, elapsed_time, error = future.result()
+                    if error:
+                        # Changing this error message requires updating CI script that checks for failed
+                        # module loading.
+                        from azure.cli.core import telemetry
+                        logger.error("Error loading command module '%s': %s", mod, error)
+                        telemetry.set_exception(exception=error, fault_type='module-load-error-' + mod,
+                                                summary='Error loading module: {}'.format(mod))
+                        logger.debug(traceback.format_exc())
+                    else:
+                        for cmd in module_command_table.values():
+                            cmd.command_source = mod
+                        self.command_table.update(module_command_table)
+                        self.command_group_table.update(module_group_table)
+
+                        logger.debug(self.item_format_string, mod, elapsed_time,
+                                     len(module_group_table), len(module_command_table))
+                        count += 1
+                        cumulative_elapsed_time += elapsed_time
+                        cumulative_group_count += len(module_group_table)
+                        cumulative_command_count += len(module_command_table)
+            elapsed_time = timeit.default_timer() - start_time_wrapper
+            print(f"*** SUT operation *** took: {elapsed_time:.6f} seconds for {len(command_modules)} modules")
             # Summary line
             logger.debug(self.item_format_string,
                          "Total ({})".format(count), cumulative_elapsed_time,

src/azure-cli-core/azure/cli/core/commands/__init__.py

@@ -517,6 +517,7 @@ def execute(self, args):
         args = _pre_command_table_create(self.cli_ctx, args)
 
         self.cli_ctx.raise_event(EVENT_INVOKER_PRE_CMD_TBL_CREATE, args=args)
+        # @NOTE: below takes forever:
         self.commands_loader.load_command_table(args)
         self.cli_ctx.raise_event(EVENT_INVOKER_PRE_CMD_TBL_TRUNCATE,
                                  load_cmd_tbl_func=self.commands_loader.load_command_table, args=args)