diff --git a/src/aaz_dev/cli/api/_cmds.py b/src/aaz_dev/cli/api/_cmds.py index ea647a41..1dff8441 100644 --- a/src/aaz_dev/cli/api/_cmds.py +++ b/src/aaz_dev/cli/api/_cmds.py @@ -1,11 +1,21 @@ import click import logging from flask import Blueprint +import os import sys +from cli.controller.az_module_manager import AzExtensionManager, AzMainManager +from command.controller.specs_manager import AAZSpecsManager +from command.controller.workspace_manager import WorkspaceManager +from command.model.configuration import CMDHelp +from swagger.controller.specs_manager import SwaggerSpecsManager +from swagger.model.specs import TypeSpecResourceProvider +from swagger.utils.source import SourceTypeEnum + from utils.config import Config +from utils.exceptions import InvalidAPIUsage -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') bp = Blueprint('cli-cmds', __name__, url_prefix='/CLI/CMDs', cli_group="cli") bp.cli.short_help = "Manage aaz commands in azure-cli and azure-cli-extensions." @@ -39,9 +49,6 @@ help="Name of the module in azure-cli or the extension in azure-cli-extensions" ) def regenerate_code(extension_or_module_name, cli_path=None, cli_extension_path=None): - from utils.config import Config - from utils.exceptions import InvalidAPIUsage - from cli.controller.az_module_manager import AzExtensionManager, AzMainManager if not cli_path and not cli_extension_path: logger.error("Please provide `--cli-path` or `--cli-extension-path`") sys.exit(1) @@ -129,11 +136,6 @@ def regenerate_code(extension_or_module_name, cli_path=None, cli_extension_path= default=Config.CLI_DEFAULT_PROFILE, ) def generate_by_swagger_tag(profile, swagger_tag, extension_or_module_name, cli_path=None, cli_extension_path=None): - from utils.config import Config - from utils.exceptions import InvalidAPIUsage - from cli.controller.az_module_manager import AzExtensionManager, AzMainManager - from swagger.controller.specs_manager import SwaggerSpecsManager - from command.controller.specs_manager import AAZSpecsManager if not Config.DEFAULT_SWAGGER_MODULE: Config.DEFAULT_SWAGGER_MODULE = "__MODULE__" @@ -197,6 +199,189 @@ def generate_by_swagger_tag(profile, swagger_tag, extension_or_module_name, cli_ sys.exit(1) +@bp.cli.command("generate", short_help="Generate code effortlessly. If the result isn't what you expected, use the UI to fine-tune it.") +@click.option( + "--cli-path", '-c', + type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True), + callback=Config.validate_and_setup_cli_path, + help="The local path of azure-cli repo. Only required when generate code to azure-cli repo." +) +@click.option( + "--cli-extension-path", '-e', + type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True), + callback=Config.validate_and_setup_cli_extension_path, + help="The local path of azure-cli-extension repo. Only required when generate code to azure-cli-extension repo." +) +@click.option( + "--swagger-path", '-s', + type=click.Path(file_okay=False, dir_okay=True, readable=True, resolve_path=True), + default=Config.SWAGGER_PATH, + callback=Config.validate_and_setup_swagger_path, + expose_value=False, + help="The local path of azure-rest-api-specs repo. Official repo is https://github.com/Azure/azure-rest-api-specs" +) +@click.option( + "--aaz-path", '-a', + type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True), + default=Config.AAZ_PATH, + required=not Config.AAZ_PATH, + callback=Config.validate_and_setup_aaz_path, + expose_value=False, + help="The local path of aaz repo." +) +@click.option( + "--spec", + required=True, + help="Folder name of the specification source." +) +@click.option( + "--module", + required=True, + help="Module name of the CLI extension target." +) +def generate(spec, module, cli_path=None, cli_extension_path=None): + def _collect_resources(spec): + module_manager = SwaggerSpecsManager().get_module_manager(Config.DEFAULT_PLANE, spec) + rps = module_manager.get_resource_providers() + + results = {} + for r in rps: + if isinstance(r, TypeSpecResourceProvider): + continue + + rp = module_manager.get_openapi_resource_provider(r.name) + tag = r.default_tag + + resource_map = rp.get_resource_map_by_tag(tag) + if not resource_map: + raise InvalidAPIUsage(f"Tag `{tag}` is not exist.") + + results[rp.name] = (resource_map, tag) + + return results + + def _normalize_resource_map(resource_map, tag): + # covert resource_map to {version: [resources]} + version_resource_map = {} + for resource_id, version_map in resource_map.items(): + v_list = list(version_map) + if len(v_list) > 1: + raise InvalidAPIUsage( + f"Tag `{tag}` contains multiple api versions of one resource", + payload={"Resource": resource_id, "versions": v_list}, + ) + + v = v_list[0] + version_resource_map.setdefault(v, []).append({"id": resource_id}) + + return version_resource_map + + def to_aaz(): + results = _collect_resources(spec) + + for rp_name, (resource_map, tag) in results.items(): + version_resource_map = _normalize_resource_map(resource_map, tag) + + ws = WorkspaceManager.new( + name=spec, + plane=Config.DEFAULT_PLANE, + folder=WorkspaceManager.IN_MEMORY, + mod_names=spec, + resource_provider=rp_name, + swagger_manager=SwaggerSpecsManager(), + aaz_manager=AAZSpecsManager(), + source=SourceTypeEnum.OpenAPI, + ) + for version, resources in version_resource_map.items(): + ws.add_new_resources_by_swagger(mod_names=spec, version=version, resources=resources) + + # complete default help + for node in ws.iter_command_tree_nodes(): + node.help = node.help or CMDHelp() + if not node.help.short: + node.help.short = f"Manage {node.names[-1]}." + + for leaf in ws.iter_command_tree_leaves(): + leaf.help = leaf.help or CMDHelp() + if not leaf.help.short: + n = leaf.names[-1] + leaf.help.short = f"{n.capitalize()} {leaf.names[-2]}" + + cfg_editor = ws.load_cfg_editor_by_command(leaf) + command = cfg_editor.find_command(*leaf.names) + leaf.examples = ws.generate_examples_by_swagger(leaf, command) + + if not ws.is_in_memory: + ws.save() + + ws.generate_to_aaz() + + def to_cli(): + results = _collect_resources(spec) + + commands_map = {} + for _, (resource_map, tag) in results.items(): + for resource_id, version_map in resource_map.items(): + v_list = list(version_map) + if len(v_list) > 1: + raise InvalidAPIUsage( + f"Tag `{tag}` contains multiple api versions of one resource", + payload={"Resource": resource_id, "versions": v_list}, + ) + + v = v_list[0] + cfg_reader = AAZSpecsManager().load_resource_cfg_reader(Config.DEFAULT_PLANE, resource_id, v) + if not cfg_reader: + logger.error(f"Command models not exist in aaz for resource: {resource_id} version: {v}.") + continue + + for cmd_names, command in cfg_reader.iter_commands(): + key = tuple(cmd_names) + if key in commands_map and commands_map[key] != command.version: + raise ValueError(f"Multi version contained for command: {''.join(cmd_names)} versions: {commands_map[key]}, {command.version}") + + commands_map[key] = command.version + + if cli_path is not None: + assert Config.CLI_PATH is not None + manager = AzMainManager() + + else: # generate ext module by default + assert Config.CLI_EXTENSION_PATH is not None + manager = AzExtensionManager() + + if not manager.has_module(module): + logger.info(f"Create cli module `{module}`.") + manager.create_new_mod(module) + + logger.info(f"Load cli module `{module}`.") + ext = manager.load_module(module) + + profile = _build_profile(Config.CLI_DEFAULT_PROFILE, commands_map) + ext.profiles[profile.name] = profile + + logger.info(f"Regenerate module `{module}`.") + manager.update_module(module, ext.profiles) + + if cli_path and cli_extension_path: + logger.error("Please provide either `--cli-path` or `--cli-extension-path`.") + raise sys.exit(1) + + spec_path = os.path.join(Config.SWAGGER_PATH, "specification", spec) + if not os.path.exists(spec_path) or not os.path.isdir(spec_path): + raise ValueError(f"Cannot find the specification name under {Config.SWAGGER_PATH}.") + + try: + to_aaz() + logger.info("✔ Finished generating AAZ model.") + to_cli() + logger.info("✔ Finished generating AAZ codes.") + + except (InvalidAPIUsage, ValueError) as err: + logger.error(err, exc_info=True) + raise sys.exit(1) + + def _build_profile(profile_name, commands_map): from cli.model.view import CLIViewProfile, CLIViewCommand, CLIViewCommandGroup profile = CLIViewProfile({ diff --git a/src/aaz_dev/cli/controller/az_atomic_profile_builder.py b/src/aaz_dev/cli/controller/az_atomic_profile_builder.py index 1264b1de..31f60154 100644 --- a/src/aaz_dev/cli/controller/az_atomic_profile_builder.py +++ b/src/aaz_dev/cli/controller/az_atomic_profile_builder.py @@ -17,7 +17,7 @@ from utils.plane import PlaneEnum from utils.case import to_camel_case, to_snake_case -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class AzAtomicProfileBuilder: @@ -248,8 +248,8 @@ def _complete_command_wait_info(cls, command_group): for rid, value in [*wait_cmd_rids.items()]: if "get_op" not in value: - logger.error(f'Failed to support wait command for resource: ' - f'Get operation with provisioning state property does not exist: {rid}') + logger.warning(f'Failed to support wait command for resource: ' + f'Get operation with provisioning state property does not exist: {rid}') del wait_cmd_rids[rid] if not wait_cmd_rids: diff --git a/src/aaz_dev/cli/controller/az_client_generator.py b/src/aaz_dev/cli/controller/az_client_generator.py index 75eda598..85119a31 100644 --- a/src/aaz_dev/cli/controller/az_client_generator.py +++ b/src/aaz_dev/cli/controller/az_client_generator.py @@ -11,7 +11,7 @@ from utils.plane import PlaneEnum from utils.case import to_camel_case -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class AzClientsGenerator: diff --git a/src/aaz_dev/cli/controller/az_command_ctx.py b/src/aaz_dev/cli/controller/az_command_ctx.py index 464e5437..5018baab 100644 --- a/src/aaz_dev/cli/controller/az_command_ctx.py +++ b/src/aaz_dev/cli/controller/az_command_ctx.py @@ -6,7 +6,7 @@ from .az_arg_group_generator import AzArgClsGenerator from .az_operation_generator import AzRequestClsGenerator, AzResponseClsGenerator -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class AzCommandCtx: diff --git a/src/aaz_dev/cli/controller/az_command_generator.py b/src/aaz_dev/cli/controller/az_command_generator.py index d0f6ffec..86e1267f 100644 --- a/src/aaz_dev/cli/controller/az_command_generator.py +++ b/src/aaz_dev/cli/controller/az_command_generator.py @@ -14,7 +14,7 @@ from .az_output_generator import AzOutputGenerator from .az_selector_generator import AzJsonSelectorGenerator -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class AzCommandGenerator: diff --git a/src/aaz_dev/cli/controller/az_module_manager.py b/src/aaz_dev/cli/controller/az_module_manager.py index 4d53646d..797ecce8 100644 --- a/src/aaz_dev/cli/controller/az_module_manager.py +++ b/src/aaz_dev/cli/controller/az_module_manager.py @@ -14,7 +14,7 @@ from utils.config import Config from collections import deque -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class AzModuleManager: @@ -36,9 +36,8 @@ def get_aaz_path(self, mod_name): raise NotImplementedError() def has_module(self, mod_name): - mod_folder = self.get_mod_path(mod_name) - if not os.path.exists(mod_folder): - #print(f"Invalid module folder: cannot find modules in: '{mod_folder}'") + mod_file = os.path.join(self.get_mod_path(mod_name), "setup.py") + if not os.path.exists(mod_file): return False return True diff --git a/src/aaz_dev/command/api/_cmds.py b/src/aaz_dev/command/api/_cmds.py index 94dd0fcd..a1947d74 100644 --- a/src/aaz_dev/command/api/_cmds.py +++ b/src/aaz_dev/command/api/_cmds.py @@ -13,7 +13,7 @@ from swagger.utils.source import SourceTypeEnum from utils.config import Config -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') bp = Blueprint('aaz-cmds', __name__, url_prefix='/AAZ/CMDs', cli_group="command-model") bp.cli.short_help = "Manage command models in aaz." diff --git a/src/aaz_dev/command/controller/specs_manager.py b/src/aaz_dev/command/controller/specs_manager.py index 324623bb..cb4360be 100644 --- a/src/aaz_dev/command/controller/specs_manager.py +++ b/src/aaz_dev/command/controller/specs_manager.py @@ -16,7 +16,7 @@ from .cfg_validator import CfgValidator from .command_tree import CMDSpecsPartialCommandTree -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class AAZSpecsManager: diff --git a/src/aaz_dev/command/controller/workspace_cfg_editor.py b/src/aaz_dev/command/controller/workspace_cfg_editor.py index 5f0b0f6b..339e779c 100644 --- a/src/aaz_dev/command/controller/workspace_cfg_editor.py +++ b/src/aaz_dev/command/controller/workspace_cfg_editor.py @@ -10,7 +10,7 @@ from .cfg_reader import CfgReader from .workspace_helper import ArgumentUpdateMixin -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class WorkspaceCfgEditor(CfgReader, ArgumentUpdateMixin): diff --git a/src/aaz_dev/command/controller/workspace_client_cfg_editor.py b/src/aaz_dev/command/controller/workspace_client_cfg_editor.py index 6a99e864..3193a983 100644 --- a/src/aaz_dev/command/controller/workspace_client_cfg_editor.py +++ b/src/aaz_dev/command/controller/workspace_client_cfg_editor.py @@ -9,7 +9,7 @@ CMDClientEndpointsByHttpOperation, CMDClientEndpoints, CMDClientEndpointCloudMetadataTemplate) -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class WorkspaceClientCfgEditor(ClientCfgReader, ArgumentUpdateMixin): diff --git a/src/aaz_dev/command/controller/workspace_manager.py b/src/aaz_dev/command/controller/workspace_manager.py index 27a58193..cd307652 100644 --- a/src/aaz_dev/command/controller/workspace_manager.py +++ b/src/aaz_dev/command/controller/workspace_manager.py @@ -20,7 +20,7 @@ from .workspace_cfg_editor import WorkspaceCfgEditor, build_endpoint_selector_for_client_config from .workspace_client_cfg_editor import WorkspaceClientCfgEditor -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class WorkspaceManager: diff --git a/src/aaz_dev/command/model/configuration/_client.py b/src/aaz_dev/command/model/configuration/_client.py index 2dfdcd69..342cb675 100644 --- a/src/aaz_dev/command/model/configuration/_client.py +++ b/src/aaz_dev/command/model/configuration/_client.py @@ -19,7 +19,7 @@ from ._resource import CMDResource -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class CMDClientAADAuthConfig(Model): diff --git a/src/aaz_dev/command/model/configuration/_command.py b/src/aaz_dev/command/model/configuration/_command.py index 7785bf80..d90ea17a 100644 --- a/src/aaz_dev/command/model/configuration/_command.py +++ b/src/aaz_dev/command/model/configuration/_command.py @@ -17,7 +17,7 @@ from ._selector_index import CMDArrayIndexBase, CMDObjectIndexBase, CMDObjectIndexDiscriminator, CMDObjectIndexAdditionalProperties from utils import exceptions -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class CMDCommand(Model): diff --git a/src/aaz_dev/command/model/configuration/_fields.py b/src/aaz_dev/command/model/configuration/_fields.py index 57af6093..fdb3b824 100644 --- a/src/aaz_dev/command/model/configuration/_fields.py +++ b/src/aaz_dev/command/model/configuration/_fields.py @@ -4,7 +4,7 @@ import logging -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class CMDBooleanField(BooleanType): diff --git a/src/aaz_dev/command/model/configuration/_schema.py b/src/aaz_dev/command/model/configuration/_schema.py index d2e3bdb1..a7ceb010 100644 --- a/src/aaz_dev/command/model/configuration/_schema.py +++ b/src/aaz_dev/command/model/configuration/_schema.py @@ -36,7 +36,7 @@ import logging import re -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class CMDSchemaEnumItem(Model): diff --git a/src/aaz_dev/swagger/api/_cmds.py b/src/aaz_dev/swagger/api/_cmds.py index 47cb908c..2921c157 100644 --- a/src/aaz_dev/swagger/api/_cmds.py +++ b/src/aaz_dev/swagger/api/_cmds.py @@ -6,7 +6,7 @@ from utils.config import Config -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') bp = Blueprint('swagger-cmds', __name__, url_prefix='/Swagger/CMDs', cli_group="swagger") bp.cli.short_help = "Manage azure-rest-api-specs/azure-rest-api-specs-pr repos." diff --git a/src/aaz_dev/swagger/controller/command_generator.py b/src/aaz_dev/swagger/controller/command_generator.py index 08f2cd64..4c418681 100644 --- a/src/aaz_dev/swagger/controller/command_generator.py +++ b/src/aaz_dev/swagger/controller/command_generator.py @@ -21,7 +21,7 @@ from utils.case import to_singular -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class _CommandGenerator(ABC): diff --git a/src/aaz_dev/swagger/model/schema/cmd_builder.py b/src/aaz_dev/swagger/model/schema/cmd_builder.py index e24ad962..c24c6942 100644 --- a/src/aaz_dev/swagger/model/schema/cmd_builder.py +++ b/src/aaz_dev/swagger/model/schema/cmd_builder.py @@ -37,7 +37,7 @@ import logging import re -logger = logging.getLogger("backend") +logger = logging.getLogger("aaz") class CMDBuilder: diff --git a/src/aaz_dev/swagger/model/schema/operation.py b/src/aaz_dev/swagger/model/schema/operation.py index d41654a3..7e06e4de 100644 --- a/src/aaz_dev/swagger/model/schema/operation.py +++ b/src/aaz_dev/swagger/model/schema/operation.py @@ -21,7 +21,7 @@ from .x_ms_odata import XmsODataField from .x_ms_pageable import XmsPageableField -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class Operation(Model, Linkable): diff --git a/src/aaz_dev/swagger/model/schema/parameter.py b/src/aaz_dev/swagger/model/schema/parameter.py index 9ff675e6..502e8fc6 100644 --- a/src/aaz_dev/swagger/model/schema/parameter.py +++ b/src/aaz_dev/swagger/model/schema/parameter.py @@ -16,7 +16,7 @@ from .schema import Schema, ReferenceSchema, schema_and_reference_schema_claim_function from .x_ms_parameter_grouping import XmsParameterGroupingField -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class ParameterBase(Model): diff --git a/src/aaz_dev/swagger/model/schema/schema.py b/src/aaz_dev/swagger/model/schema/schema.py index 18e38b51..84c2d107 100644 --- a/src/aaz_dev/swagger/model/schema/schema.py +++ b/src/aaz_dev/swagger/model/schema/schema.py @@ -22,7 +22,7 @@ from .x_ms_enum import XmsEnumField from .xml import XML -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') def schema_and_reference_schema_claim_function(_, data): diff --git a/src/aaz_dev/swagger/model/specs/_resource.py b/src/aaz_dev/swagger/model/specs/_resource.py index bd95e0e5..98a3fc5c 100644 --- a/src/aaz_dev/swagger/model/specs/_resource.py +++ b/src/aaz_dev/swagger/model/specs/_resource.py @@ -12,7 +12,7 @@ from swagger.utils import exceptions from utils.case import to_singular -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class Resource: diff --git a/src/aaz_dev/swagger/model/specs/_resource_provider.py b/src/aaz_dev/swagger/model/specs/_resource_provider.py index 0ae26ae9..e24ed4be 100644 --- a/src/aaz_dev/swagger/model/specs/_resource_provider.py +++ b/src/aaz_dev/swagger/model/specs/_resource_provider.py @@ -3,6 +3,7 @@ import logging import os import re +import sys from collections import OrderedDict import yaml @@ -11,7 +12,7 @@ from ._resource import Resource, ResourceVersion from ._utils import map_path_2_repo from utils.readme_helper import parse_readme_file -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class OpenAPIResourceProvider: @@ -25,6 +26,7 @@ def __init__(self, name, folder_path, readme_paths, swagger_module): if not readme_paths: logger.warning(f"MissReadmeFile: {self} : {map_path_2_repo(folder_path)}") self._tags = None + self._default_tag = None self._resource_map = None self._ignore_resources = {f'/providers/{self.name}/operations'.lower(), } @@ -86,6 +88,21 @@ def tags(self): self._tags = self._parse_readme_input_file_tags() return self._tags + @property + def default_tag(self): + if self._default_tag is None: + with open(self._readme_paths[0], "r", encoding="utf-8") as f: + content = f.read() + + try: + self._default_tag = re.findall(r"tag:\s*(.+)", content)[0] + + except IndexError: + logger.error(f"Cannot find default tag in resource provider: {self.name}.", exc_info=True) + raise sys.exit(1) + + return self._default_tag + def _parse_readme_input_file_tags(self): tags = {} if not self._readme_paths: diff --git a/src/aaz_dev/swagger/model/specs/_swagger_loader.py b/src/aaz_dev/swagger/model/specs/_swagger_loader.py index 43b75084..da0c94cd 100644 --- a/src/aaz_dev/swagger/model/specs/_swagger_loader.py +++ b/src/aaz_dev/swagger/model/specs/_swagger_loader.py @@ -5,7 +5,7 @@ from swagger.utils import exceptions -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class SwaggerLoader: diff --git a/src/aaz_dev/swagger/model/specs/_typespec_helper.py b/src/aaz_dev/swagger/model/specs/_typespec_helper.py index 1fc7259c..c91645cf 100644 --- a/src/aaz_dev/swagger/model/specs/_typespec_helper.py +++ b/src/aaz_dev/swagger/model/specs/_typespec_helper.py @@ -2,7 +2,7 @@ import logging import re -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') class TypeSpecHelper: diff --git a/src/aaz_dev/swagger/model/specs/_utils.py b/src/aaz_dev/swagger/model/specs/_utils.py index 538b9dff..5199c521 100644 --- a/src/aaz_dev/swagger/model/specs/_utils.py +++ b/src/aaz_dev/swagger/model/specs/_utils.py @@ -2,7 +2,7 @@ import logging import re -logger = logging.getLogger('backend') +logger = logging.getLogger('aaz') def map_path_2_repo(path):