update mgmt docs build to not rely on hardcoded mappings (#34482)

* hardcoding everything wip, should able to compute automatically

* update the code to not use hardcoded package_service_mapping

* make sure we preserve ordering of latest --> oldest

* update copyright year

Author: kristapratico

doc/sphinx/conf.py

@@ -60,9 +60,9 @@
 
 intersphinx_mapping = {
     # Dependencies
-    'python': ('https://docs.python.org/3.11', ('/usr/share/doc/python3-doc/html/objects.inv', None)),
-    'requests': ('https://requests.kennethreitz.org/en/master/', None),
-    'aiohttp': ('https://aiohttp.readthedocs.io/en/stable/', None),
+    'python': ('https://docs.python.org/3.11', None),
+    'requests': ('https://requests.kennethreitz.org/en/latest/', None),
+    'aiohttp': ('https://docs.aiohttp.org/en/stable/', None),
     'trio': ('https://trio.readthedocs.io/en/stable/', None),
     'msal': ('https://msal-python.readthedocs.io/en/latest/', None),
     # Azure packages
@@ -87,7 +87,7 @@
 
 # General information about the project.
 project = u'Azure SDK for Python'
-copyright = u'2019, Microsoft'
+copyright = u'2024, Microsoft'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

doc/sphinx/generate_doc.py

@@ -5,10 +5,8 @@
 import os
 import glob
 
-import typing
 from typing import Dict, List
 
-CONFIG_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "package_service_mapping.json")
 GENERATED_PACKAGES_LIST_FILE = "toc_tree.rst"
 
 _LOGGER = logging.getLogger(__name__)
@@ -73,117 +71,112 @@ def make_title(title):
 
 """
 
-# Update the code to compute this list automatically
-MULTIAPI_VERSION_NAMESPACE = [
-    "azure.mgmt.storage",
-    "azure.mgmt.web",
-    "azure.mgmt.network",
-    "azure.mgmt.compute",
-    "azure.mgmt.containerregistry",
-    "azure.mgmt.containerservice",
-    "azure.mgmt.dns",
-    "azure.mgmt.eventhub",
-    "azure.mgmt.resource.resources",
-    "azure.mgmt.resource.features",
-    "azure.mgmt.resource.links",
-    "azure.mgmt.resource.locks",
-    "azure.mgmt.resource.policy",
-    "azure.mgmt.resource.subscriptions",
-]
-
-
-def get_updated_config(config_path: str, package_root: str) -> Dict[str, Dict[str, typing.Union[str, List[str]]]]:
-    with Path(config_path).open() as config_fd:
-        config = json.load(config_fd)
 
-    namespace_folders = os.path.basename(package_root).split("-")
-    package_name = "-".join(namespace_folders)
-    namespace = ".".join(namespace_folders)
-
-    api_directory = os.path.join(package_root, *namespace_folders)
+def get_valid_versions(api_directory: str) -> List[str]:
     glob_path = os.path.join(api_directory, "v20*/")
-    valid_versions = glob.glob(glob_path)
+    return glob.glob(glob_path)
 
-    for version_path in valid_versions:
-        api_version = os.path.relpath(version_path, start=api_directory)
-        full_namespace = namespace + f".{api_version}"
 
-        if package_name in config:
-            if "namespaces" in config[package_name]:
-                ns = config[package_name]["namespaces"]
-                if ns:
-                    if full_namespace not in ns:
-                        ns.append(full_namespace)
+def is_subnamespace(version: str) -> bool:
+    return not version.startswith("v20")
 
-    return config
 
+def get_package_namespaces(package_root: str) -> List[str]:
+    namespace_folders = os.path.basename(package_root).split("-")
+    namespace = ".".join(namespace_folders)
 
-def generate_doc(config_path: str, output_directory: str = "./ref/", package_root: str = None) -> None:
-    multiapi_found_apiversion = {}
-    # we are handed a directory that looks like <path-to-repo>/sdk/containerservice/azure-mgmt-containerservice/
-    project_pattern = os.path.basename(package_root).replace("-", ".")
-    rst_path_template = os.path.join(output_directory, "{}.rst")
-    rst_namespace_template = os.path.join(output_directory, "{}.{}.rst")
+    # add top namespace
+    namespaces = {namespace: []}
 
-    config = get_updated_config(config_path, package_root)
+    api_directory = os.path.join(package_root, *namespace_folders)
+    valid_versions = get_valid_versions(api_directory)
 
-    package_list_path = []
+    # check for subnamespaces like azure.mgmt.resource.locks
+    if not valid_versions:
+        subnamespaces = glob.glob(f"{api_directory}/*/")
+        for snp_path in subnamespaces:
+            versions = get_valid_versions(snp_path)
+            valid_versions.extend(versions)
 
-    namespaces = [n for pack in config.values() for n in pack.get("namespaces", {})]
+    for version_path in valid_versions:
+        version = os.path.relpath(version_path, start=api_directory)
+        if is_subnamespace(version):
+            subnamespace_name, api_version = version.split("/")
+            full_namespace = ".".join([namespace, subnamespace_name, api_version])
+            subnamespace = ".".join([namespace, subnamespace_name])
+            if subnamespace not in namespaces[namespace]:
+                namespaces[namespace].append(subnamespace)
+            namespaces.setdefault(subnamespace, []).append(full_namespace)
+        else:
+            full_namespace = ".".join([namespace, version])
+            namespaces[namespace].append(full_namespace)
 
-    for namespace in namespaces:
-        if project_pattern and not namespace.startswith(project_pattern):
-            continue
+    return namespaces
 
-        _LOGGER.info("Working on %s", namespace)
 
-        rst_path = rst_path_template.format(namespace)
-        with Path(rst_path).open("w") as rst_file:
-            rst_file.write(PACKAGE_TEMPLATE.format(title=make_title(namespace + " package"), namespace=namespace))
+def write_rst(version: str, rst_path_template: str, rst_namespace_template: str, package_list_path: List[str]) -> None:
+    rst_path = rst_path_template.format(version)
+    with Path(rst_path).open("w") as rst_file:
+        rst_file.write(PACKAGE_TEMPLATE.format(title=make_title(version + " package"), namespace=version))
 
-        for module in ["operations", "models"]:
-            with Path(rst_namespace_template.format(namespace, module)).open("w") as rst_file:
-                rst_file.write(
-                    SUBMODULE_TEMPLATE.format(
-                        title=make_title(namespace + "." + module + " module"), namespace=namespace, submodule=module
-                    )
+    for module in ["operations", "models"]:
+        with Path(rst_namespace_template.format(version, module)).open("w") as rst_file:
+            rst_file.write(
+                SUBMODULE_TEMPLATE.format(
+                    title=make_title(version + "." + module + " module"), namespace=version, submodule=module
                 )
+            )
+    package_list_path.append(rst_path)
 
-        for multiapi_namespace in MULTIAPI_VERSION_NAMESPACE:
-            length = len(multiapi_namespace.split("."))
-            if namespace.split(".")[0:length] == multiapi_namespace.split(".")[0:length]:
-                _LOGGER.info("MultiAPI namespace on %s", multiapi_namespace)
-                api_package = namespace.split(multiapi_namespace + ".")[1]
-                multiapi_found_apiversion.setdefault(multiapi_namespace, []).append(api_package)
-                break
-        else:
-            package_list_path.append(rst_path)
 
-    for multiapi_namespace, apilist in multiapi_found_apiversion.items():
-        apilist.sort()
-        apilist.reverse()
-        rst_path = rst_path_template.format(multiapi_namespace)
-        with Path(rst_path).open("w") as rst_file:
-            rst_file.write(
-                MULTIAPI_VERSION_PACKAGE_TEMPLATE.format(
-                    title=make_title(multiapi_namespace + " package"), namespace=multiapi_namespace
-                )
+def write_multiapi_rst(namespace: str, versions: List[str], rst_path_template: str, package_list_path: List[str]) -> None:
+    rst_path = rst_path_template.format(namespace)
+    with Path(rst_path).open("w") as rst_file:
+        rst_file.write(
+            MULTIAPI_VERSION_PACKAGE_TEMPLATE.format(
+                title=make_title(namespace + " package"), namespace=namespace
             )
-            for version in apilist:
-                rst_file.write("   {namespace}.{version}\n".format(namespace=multiapi_namespace, version=version))
-        package_list_path.append(rst_path)
+        )
+        for version in versions:
+            rst_file.write("   {version}\n".format(version=version))
+    package_list_path.append(rst_path)
 
-    # now handle the packages from data plane that we want to be present properly sorted in the list of packages
-    for package in config.keys():
-        if "manually_generated" in config[package] and config[package]["manually_generated"] == True:
-            package_list_path.append(rst_path_template.format(package.replace("-", ".")))
 
+def write_toc_tree(package_list_path: List[str]) -> None:
     package_list_path.sort()
     with Path(GENERATED_PACKAGES_LIST_FILE).open("w") as generate_file_list_fd:
         lines_to_write = "\n".join(["  " + package for package in package_list_path])
         generate_file_list_fd.write(RST_AUTODOC_TOCTREE.format(generated_packages=lines_to_write))
 
 
+def generate_doc(output_directory: str = "./ref/", package_root: str = None) -> None:
+    # we are handed a directory that looks like <path-to-repo>/sdk/containerservice/azure-mgmt-containerservice/
+    rst_path_template = os.path.join(output_directory, "{}.rst")
+    rst_namespace_template = os.path.join(output_directory, "{}.{}.rst")
+
+    namespaces = get_package_namespaces(package_root)
+
+    package_list_path = []
+
+    for namespace, multi_api_versions in namespaces.items():
+        _LOGGER.info("Working on %s", namespace)
+        if not multi_api_versions:
+            write_rst(namespace, rst_path_template, rst_namespace_template, package_list_path)
+            continue
+
+        multi_api_versions.sort(reverse=True)
+        write_multiapi_rst(namespace, multi_api_versions, rst_path_template, package_list_path)
+
+        for version in multi_api_versions:
+            if is_subnamespace(version.split(".")[-1]):
+                # subnamespace already handled in write_multiapi_rst
+                continue
+            _LOGGER.info("Working on %s", version)
+            write_rst(version, rst_path_template, rst_namespace_template, package_list_path)
+
+    write_toc_tree(package_list_path)
+
+
 def main():
     parser = argparse.ArgumentParser(description="Generate sphinx api stubs for one or multiple management packages.")
     parser.add_argument(
@@ -192,13 +185,6 @@ def main():
         dest="project",
         help="Want to target a specific management package? Pass the targeted package root to this argument.",
     )
-    parser.add_argument(
-        "--config",
-        "-c",
-        dest="config_path",
-        default=CONFIG_FILE,
-        help="The JSON configuration format path [default: %(default)s]",
-    )
     parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbosity in INFO mode")
     parser.add_argument("--debug", dest="debug", action="store_true", help="Verbosity in DEBUG mode")
     parser.add_argument(
@@ -212,7 +198,7 @@ def main():
         logging.basicConfig()
         main_logger.setLevel(logging.DEBUG if args.debug else logging.INFO)
 
-    generate_doc(args.config_path, args.output_directory, args.project)
+    generate_doc(args.output_directory, args.project)
 
 
 if __name__ == "__main__":