|
| 1 | +import os |
| 2 | +import re |
| 3 | +import json |
| 4 | +import logging |
| 5 | +import subprocess |
| 6 | + |
| 7 | +import yaml |
| 8 | +from pathlib import Path |
| 9 | +from typing import Any, Set, Dict |
| 10 | +from github import Github |
| 11 | +from tempfile import TemporaryDirectory |
| 12 | +import importlib.util |
| 13 | + |
| 14 | +from github.GithubException import UnknownObjectException |
| 15 | + |
| 16 | +_LOG = logging.getLogger() |
| 17 | + |
| 18 | +SOURCE_FILE = { |
| 19 | + "azure-cli": "https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/profiles/_shared.py", |
| 20 | + "rest-api-profiles": "https://github.com/Azure/azure-rest-api-specs/tree/main/profiles/definitions", |
| 21 | + "rest-api-profile": "https://github.com/Azure/azure-rest-api-specs/tree/main/profile", |
| 22 | + "rest-api-specification": "https://github.com/Azure/azure-rest-api-specs/tree/main/specification", |
| 23 | +} |
| 24 | + |
| 25 | + |
| 26 | +class CollectApiVersion: |
| 27 | + """ |
| 28 | + This class can collect api-version that may be used by azure stack |
| 29 | + """ |
| 30 | + |
| 31 | + def __init__(self): |
| 32 | + self.github = Github(os.getenv("TOKEN")) |
| 33 | + self.rest_repo = self.github.get_repo("Azure/azure-rest-api-specs") |
| 34 | + self.package_api_version = {} |
| 35 | + self.multi_api_version_from_profiles = {} |
| 36 | + self.multi_api_version_from_profile = {} |
| 37 | + self.provider_mapping_package = {} |
| 38 | + self.output_files = { |
| 39 | + "package_api_version": "package_api_version_from_cli.json", |
| 40 | + "multi_api_version_from_profiles": "package_api_version_from_profiles.json", |
| 41 | + "multi_api_version_from_profile": "package_api_version_from_profile.json", |
| 42 | + } |
| 43 | + |
| 44 | + def get_api_version_from_azure_cli(self): |
| 45 | + # read content from github |
| 46 | + url_path = SOURCE_FILE["azure-cli"] |
| 47 | + cli_repo = self.github.get_repo("Azure/azure-cli") |
| 48 | + git_path = url_path.split("dev/")[-1] |
| 49 | + file_content = cli_repo.get_contents(git_path) |
| 50 | + |
| 51 | + # write to temp file |
| 52 | + temp_dir = TemporaryDirectory() |
| 53 | + file_name = str(Path(f"{temp_dir.name}/cli.py")) |
| 54 | + with open(file_name, "wb") as file_in: |
| 55 | + file_in.write(file_content.decoded_content) |
| 56 | + |
| 57 | + # import target object that contains api-version |
| 58 | + module_name = "AZURE_API_PROFILES" |
| 59 | + spec = importlib.util.spec_from_file_location(module_name, file_name) |
| 60 | + foo = importlib.util.module_from_spec(spec) |
| 61 | + spec.loader.exec_module(foo) |
| 62 | + version_dict = getattr(foo, module_name) |
| 63 | + |
| 64 | + # extract api version |
| 65 | + for info in version_dict.values(): |
| 66 | + for package_info in info: |
| 67 | + package_name = package_info.import_prefix |
| 68 | + if re.search("azure.mgmt.", package_name): |
| 69 | + api_version = self.extract_api_version(info[package_info]) |
| 70 | + # eg: change them like azure.mgmt.resource to azure-mgmt-resource |
| 71 | + package_name = package_name.replace(".", "-") |
| 72 | + if package_name not in self.package_api_version: |
| 73 | + self.package_api_version[package_name] = set() |
| 74 | + self.package_api_version[package_name].update(api_version) |
| 75 | + |
| 76 | + def get_multiapi_from_rest_api(self): |
| 77 | + # map provider to package name, like: {'microsoft.insights': {'azure-mgmt-applicationinsights'}} |
| 78 | + url_path = SOURCE_FILE["rest-api-specification"] |
| 79 | + git_path = url_path.split("main/")[-1] |
| 80 | + service_paths = self.rest_repo.get_contents(git_path) |
| 81 | + packge_pattern = re.compile(b"package-name: (azure-mgmt-.*?)\n") |
| 82 | + for service_path in service_paths: |
| 83 | + try: |
| 84 | + resource_manager = self.rest_repo.get_contents(f"{service_path.path}/resource-manager") |
| 85 | + except UnknownObjectException: |
| 86 | + continue |
| 87 | + package_name, providers, multi_api_readme_python = "", set(), False |
| 88 | + for resource in resource_manager: |
| 89 | + if "Microsoft." in resource.name: |
| 90 | + providers.add(resource.name.lower()) |
| 91 | + if "readme.python.md" in resource.name: |
| 92 | + if b"multiapiscript: true" not in resource.decoded_content: |
| 93 | + break |
| 94 | + multi_api_readme_python = True |
| 95 | + package_name_line = re.search(packge_pattern, resource.decoded_content) |
| 96 | + package_name = package_name_line.groups()[0].decode(encoding="utf-8") |
| 97 | + if not multi_api_readme_python: |
| 98 | + continue |
| 99 | + for n in providers: |
| 100 | + if not self.provider_mapping_package.get(n): |
| 101 | + self.provider_mapping_package[n] = {package_name, } |
| 102 | + else: |
| 103 | + self.provider_mapping_package[n].add(package_name) |
| 104 | + |
| 105 | + def find_versions_from_json(self, provider: str, version: Dict[str, Any], multi_api_version: Dict): |
| 106 | + if self.provider_mapping_package.get(provider): |
| 107 | + for p in self.provider_mapping_package.get(provider): |
| 108 | + if not multi_api_version.get(p): |
| 109 | + multi_api_version[p] = set(version.keys()) |
| 110 | + else: |
| 111 | + multi_api_version[p].update(set(version.keys())) |
| 112 | + |
| 113 | + def get_api_version_from_rest_api_profiles(self): |
| 114 | + self.get_multiapi_from_rest_api() |
| 115 | + # Find api version mapping to {'azure-mgmt-msi': {'2018-11-30'}} |
| 116 | + url_path = SOURCE_FILE["rest-api-profiles"] |
| 117 | + git_path = url_path.split("main/")[-1] |
| 118 | + file_paths = self.rest_repo.get_contents(git_path) |
| 119 | + for file in file_paths: |
| 120 | + file_name, file_contents = file.name.replace(".md", ""), file.decoded_content |
| 121 | + profiles_content = str(file_contents).split("profiles:")[1].split("operations:")[0] |
| 122 | + profiles_content = profiles_content.strip(r"\n").strip().replace(r"\n", "\n") |
| 123 | + # Convert to JSON format |
| 124 | + content_dict = yaml.load(profiles_content, Loader=yaml.FullLoader) |
| 125 | + for provider, version in content_dict[file_name]["resources"].items(): |
| 126 | + self.find_versions_from_json(provider, version, self.multi_api_version_from_profiles) |
| 127 | + |
| 128 | + def get_api_version_from_rest_api_profile(self): |
| 129 | + if not self.provider_mapping_package: |
| 130 | + self.get_multiapi_from_rest_api() |
| 131 | + # map package name to api version like {'azure-mgmt-msi': {'2018-11-30'}} |
| 132 | + url_path = SOURCE_FILE["rest-api-profile"] |
| 133 | + git_path = url_path.split("main/")[-1] |
| 134 | + file_paths = self.rest_repo.get_contents(git_path) |
| 135 | + for file in file_paths: |
| 136 | + if ".json" not in file.name: |
| 137 | + continue |
| 138 | + file_name, file_contents = (file.name.replace(".json", ""), file.decoded_content) |
| 139 | + content_dict = json.loads(file_contents.decode()) |
| 140 | + resource_manager = "resource-manager" if content_dict.get("resource-manager") else "resourcemanager" |
| 141 | + for provider, version in content_dict[resource_manager].items(): |
| 142 | + self.find_versions_from_json(provider, version, self.multi_api_version_from_profile) |
| 143 | + |
| 144 | + @staticmethod |
| 145 | + def extract_api_version(api_version_info: Any) -> Set[str]: |
| 146 | + # convert to string |
| 147 | + if isinstance(api_version_info, str): |
| 148 | + api_version = api_version_info |
| 149 | + else: |
| 150 | + api_version = api_version_info.default_api_version + str(api_version_info.profile) |
| 151 | + |
| 152 | + return set(re.findall("\d{4}-\d{2}-\d{2}[-a-z]*", api_version)) |
| 153 | + |
| 154 | + @staticmethod |
| 155 | + def write_file(file_name, content): |
| 156 | + json_out = {k: sorted(content[k], reverse=True) for k in content} |
| 157 | + with open(file_name, "w") as file_out: |
| 158 | + json.dump(json_out, file_out, indent=4) |
| 159 | + |
| 160 | + def output(self): |
| 161 | + # output service and api version from cli or profiles or profile |
| 162 | + for k, v in self.output_files.items(): |
| 163 | + self.write_file(v, getattr(self, k)) |
| 164 | + # merge multi_api_version_from_profiles to package_api_version |
| 165 | + for k, v in self.multi_api_version_from_profiles.items(): |
| 166 | + self.package_api_version[k] = self.package_api_version[k] | v if self.package_api_version.get(k) else v |
| 167 | + # merge multi_api_version_from_profile to package_api_version |
| 168 | + for k, v in self.multi_api_version_from_profile.items(): |
| 169 | + self.package_api_version[k] = self.package_api_version[k] | v if self.package_api_version.get(k) else v |
| 170 | + |
| 171 | + # output all apiversion |
| 172 | + self.write_file("package_api_version_all.json", self.package_api_version) |
| 173 | + |
| 174 | + def run(self): |
| 175 | + self.get_api_version_from_azure_cli() |
| 176 | + self.get_api_version_from_rest_api_profiles() |
| 177 | + self.get_api_version_from_rest_api_profile() |
| 178 | + self.output() |
| 179 | + |
| 180 | + |
| 181 | +def print_exec(cmd): |
| 182 | + _LOG.info("==" + cmd + " ==\n") |
| 183 | + subprocess.call(cmd, shell=True) |
| 184 | + |
| 185 | + |
| 186 | +def print_check(cmd): |
| 187 | + _LOG.info("==" + cmd + " ==\n") |
| 188 | + subprocess.check_call(cmd, shell=True) |
| 189 | + |
| 190 | + |
| 191 | +def upload_to_github(): |
| 192 | + print_exec("git add .") |
| 193 | + print_exec('git commit -m "update json files"') |
| 194 | + print_check("git push origin HEAD -f") |
| 195 | + |
| 196 | + |
| 197 | +if __name__ == "__main__": |
| 198 | + main_logger = logging.getLogger() |
| 199 | + logging.basicConfig() |
| 200 | + main_logger.setLevel(logging.INFO) |
| 201 | + |
| 202 | + instance = CollectApiVersion() |
| 203 | + instance.run() |
| 204 | + upload_to_github() |
0 commit comments