[sdk generation pipeline] remove logic related with multiapi (#43640)

* remove logic related with multiapi

* format

* fix

Author: msyyc

eng/tools/azure-sdk-tools/packaging_tools/generate_utils.py

@@ -72,20 +72,6 @@ def del_outdated_generated_files(tsp: str):
             _LOGGER.info(f"delete outdated {item} successfully")
 
 
-def check_api_version_in_subfolder(sdk_code_path: str):
-    folders = glob(f"{sdk_code_path}/**/_configuration.py", recursive=True)
-    configs = [str(Path(f)) for f in folders if re.compile("v\d{4}_\d{2}_\d{2}").search(f)]
-    if configs:
-        result = []
-        for config in configs:
-            with open(config, "r") as file_in:
-                content = file_in.readlines()
-                if "self.api_version = api_version" not in "".join(content):
-                    result.append(config)
-        if result:
-            raise Exception("Found files that do not set api_version: \n" + "\n".join(result))
-
-
 def dpg_relative_folder(spec_folder: str) -> str:
     return ("../" * 4) + spec_folder + "/"
 

eng/tools/azure-sdk-tools/packaging_tools/package_utils.py

@@ -1,3 +1,4 @@
+import sys
 import os
 import ast
 import time
@@ -25,7 +26,7 @@
 # prefolder: "sdk/compute"; name: "azure-mgmt-compute"
 def create_package(prefolder, name):
     absdirpath = Path(prefolder, name).absolute()
-    check_call(["python", "-m", "build"], cwd=absdirpath)
+    check_call([sys.executable, "-m", "build"], cwd=absdirpath)
 
 
 @return_origin_path

eng/tools/azure-sdk-tools/packaging_tools/sdk_generator.py

@@ -1,11 +1,11 @@
 import sys
 import time
-from typing import List, Dict, Any
+from typing import List, Any
 import argparse
 import json
 import logging
 from pathlib import Path
-from subprocess import check_call, getoutput
+from subprocess import check_call
 import shutil
 import re
 import os
@@ -33,9 +33,6 @@
     gen_dpg,
     dpg_relative_folder,
     gen_typespec,
-    return_origin_path,
-    check_api_version_in_subfolder,
-    call_build_config,
     del_outdated_generated_files,
 )
 from .conf import CONF_NAME
@@ -53,13 +50,6 @@ def execute_func_with_timeout(func, timeout: int = 900) -> Any:
     return multiprocessing.Pool(processes=1).apply_async(func).get(timeout)
 
 
-def is_multiapi_package(python_md_content: List[str]) -> bool:
-    for line in python_md_content:
-        if re.findall(r"\s*multiapi\s*:\s*true", line):
-            return True
-    return False
-
-
 # return relative path like: network/azure-mgmt-network
 def extract_sdk_folder(python_md: List[str]) -> str:
     pattern = ["$(python-sdks-folder)", "azure-mgmt-"]
@@ -84,7 +74,6 @@ def get_readme_python_content(readme: str) -> List[str]:
 def del_outdated_files(readme: str):
     content = get_readme_python_content(readme)
     sdk_folder = extract_sdk_folder(content)
-    is_multiapi = is_multiapi_package(content)
     if sdk_folder:
         # remove tsp-location.yaml
         tsp_location = Path(f"sdk/{sdk_folder}/tsp-location.yaml")
@@ -94,127 +83,16 @@ def del_outdated_files(readme: str):
         # remove generated_samples
         sample_folder = Path(f"sdk/{sdk_folder}/generated_samples")
         if sample_folder.exists():
-            # rdbms is generated from different swagger folder;multiapi package may don't generate every time
-            if "azure-mgmt-rdbms" not in str(sample_folder) and not is_multiapi:
+            # rdbms is generated from different swagger folder
+            if "azure-mgmt-rdbms" not in str(sample_folder):
                 shutil.rmtree(sample_folder)
                 _LOGGER.info(f"remove sample folder: {sample_folder}")
             else:
-                _LOGGER.info(f"we don't remove sample folder for rdbms or multiapi package")
+                _LOGGER.info(f"we don't remove sample folder for rdbms")
         else:
             _LOGGER.info(f"sample folder does not exist: {sample_folder}")
     else:
-        _LOGGER.info(f"do not find valid sdk_folder in {python_readme}")
-
-
-# look for fines in tag like:
-# ``` yaml $(tag) == 'package-2023-05-01-preview-only'
-# input-file:
-# - Microsoft.Insights/preview/2023-05-01-preview/tenantActionGroups_API.json
-# ```
-def get_related_swagger(readme_content: List[str], tag: str) -> List[str]:
-    result = []
-    for idx in range(len(readme_content)):
-        line = readme_content[idx]
-        if tag in line and "```" in line and "tag" in line and "==" in line and "yaml" in line:
-            idx += 1
-            while idx < len(readme_content):
-                if "```" in readme_content[idx]:
-                    break
-                if ".json" in readme_content[idx] and (
-                    re.compile(r"\d{4}-\d{1,2}-\d{1,2}").findall(readme_content[idx])
-                    or "Microsoft." in readme_content[idx]
-                ):
-                    result.append(readme_content[idx].strip("\n -"))
-                idx += 1
-            break
-    return result
-
-
-def get_last_commit_info(files: List[str]) -> str:
-    result = [getoutput(f'git log -1 --pretty="format:%ai %H" {f}').strip("\n ") + " " + f for f in files]
-    result.sort()
-    return result[-1]
-
-
-# input_readme: "specification/paloaltonetworks/resource-manager/readme.md"
-# source: content of readme.python.md
-# work directory is in root folder of azure-rest-api-specs
-@return_origin_path
-def choose_tag_and_update_meta(
-    idx: int, source: List[str], target: List[str], input_readme: str, meta: Dict[str, Any], need_regenerate: bool
-) -> int:
-    os.chdir(str(Path(input_readme).parent))
-    with open("readme.md", "r") as file_in:
-        readme_content = file_in.readlines()
-
-    while idx < len(source):
-        if "```" in source[idx]:
-            break
-        if "tag:" in source[idx]:
-            tag = source[idx].split("tag:")[-1].strip("\n ")
-            related_files = get_related_swagger(readme_content, tag)
-            if related_files:
-                commit_info = get_last_commit_info(related_files)
-                recorded_info = meta.get(tag, "")
-                # there may be new commit after last release
-                if need_regenerate or commit_info > recorded_info:
-                    _LOGGER.info(f"update tag: {tag} with commit info {commit_info}")
-                    meta[tag] = commit_info
-                    target.append(source[idx])
-                else:
-                    _LOGGER.info(f"skip tag: {tag} since commit info doesn't change")
-            else:
-                _LOGGER.warning(f"do not find related swagger for tag: {tag}")
-        else:
-            target.append(source[idx])
-        idx += 1
-    return idx
-
-
-def extract_version_info(config: Dict[str, Any]) -> str:
-    autorest_version = config.get("autorest", "")
-    autorest_modelerfour_version = config.get("use", [])
-    return autorest_version + "".join(autorest_modelerfour_version)
-
-
-def if_need_regenerate(meta: Dict[str, Any]) -> bool:
-    with open(str(Path("../azure-sdk-for-python", CONFIG_FILE)), "r") as file_in:
-        config = json.load(file_in)
-    current_info = config["meta"]["autorest_options"]["version"] + "".join(
-        sorted(config["meta"]["autorest_options"]["use"])
-    )
-    recorded_info = meta["autorest"] + "".join(sorted(meta["use"]))
-    return recorded_info != current_info
-
-
-# spec_folder: "../azure-rest-api-specs"
-# input_readme: "specification/paloaltonetworks/resource-manager/readme.md"
-@return_origin_path
-def need_regen_for_multiapi_package(spec_folder: str, input_readme: str) -> bool:
-    os.chdir(spec_folder)
-    python_readme = (Path(input_readme).parent / "readme.python.md").absolute()
-    if not python_readme.exists():
-        _LOGGER.info(f"do not find python configuration: {python_readme}")
-        return False
-
-    with open(python_readme, "r") as file_in:
-        python_md_content = file_in.readlines()
-    is_multiapi = is_multiapi_package(python_md_content)
-    if not is_multiapi:
-        _LOGGER.info(f"do not find multiapi configuration in {python_readme}")
-        return False
-
-    after_handle = []
-    for idx in range(len(python_md_content)):
-        if re.findall(r"\s*clear-output-folder\s*:\s*true\s*", python_md_content[idx]):
-            continue
-        if re.findall(r"\s*-\s*tag\s*:", python_md_content[idx]):
-            continue
-        after_handle.append(python_md_content[idx])
-
-    with open(python_readme, "w") as file_out:
-        file_out.writelines(after_handle)
-    return True
+        _LOGGER.info(f"do not find valid sdk_folder in readme.python.md")
 
 
 def main(generate_input, generate_output):
@@ -232,6 +110,7 @@ def main(generate_input, generate_output):
     run_in_pipeline = data.get("runMode") is not None
     for input_type, readme_or_tsp in readme_and_tsp:
         _LOGGER.info(f"[CODEGEN]({readme_or_tsp})codegen begin")
+        config = None
         try:
             code_generation_start_time = time.time()
             if input_type == "relatedTypeSpecProjectFolder":
@@ -248,8 +127,7 @@ def main(generate_input, generate_output):
             elif "resource-manager" in readme_or_tsp:
                 relative_path_readme = str(Path(spec_folder, readme_or_tsp))
                 del_outdated_files(relative_path_readme)
-                generate_mgmt = partial(
-                    generate,
+                config = generate(
                     CONFIG_FILE,
                     sdk_folder,
                     [],
@@ -258,9 +136,6 @@ def main(generate_input, generate_output):
                     force_generation=True,
                     python_tag=python_tag,
                 )
-                config = generate_mgmt()
-                if need_regen_for_multiapi_package(spec_folder, readme_or_tsp):
-                    generate_mgmt()
             else:
                 config = gen_dpg(readme_or_tsp, data.get("autorestConfig", ""), dpg_relative_folder(spec_folder))
             _LOGGER.info(f"code generation cost time: {int(time.time() - code_generation_start_time)} seconds")
@@ -296,8 +171,6 @@ def main(generate_input, generate_output):
                     package_entry["path"] = [folder_name]
                     package_entry[spec_word] = [readme_or_tsp]
                     package_entry["tagIsStable"] = not judge_tag_preview(sdk_code_path, package_name)
-                    readme_python_content = get_readme_python_content(str(Path(spec_folder) / readme_or_tsp))
-                    package_entry["isMultiapi"] = is_multiapi_package(readme_python_content)
                     package_entry["targetReleaseDate"] = data.get("targetReleaseDate", "")
                     package_entry["allowInvalidNextVersion"] = data.get("allowInvalidNextVersion", False)
                     result[package_name] = package_entry
@@ -322,15 +195,18 @@ def main(generate_input, generate_output):
 
             # Update metadata
             try:
-                update_servicemetadata(
-                    sdk_folder,
-                    data,
-                    config,
-                    folder_name,
-                    package_name,
-                    spec_folder,
-                    readme_or_tsp,
-                )
+                if config is not None:
+                    update_servicemetadata(
+                        sdk_folder,
+                        data,
+                        config,
+                        folder_name,
+                        package_name,
+                        spec_folder,
+                        readme_or_tsp,
+                    )
+                else:
+                    _LOGGER.warning(f"Skip metadata update for {package_name} as config is not available")
             except Exception as e:
                 _LOGGER.warning(f"Fail to update meta: {str(e)}")
 
@@ -343,15 +219,6 @@ def main(generate_input, generate_output):
             except Exception as e:
                 _LOGGER.warning(f"Fail to setup package {package_name} in {readme_or_tsp}: {str(e)}")
 
-            # check whether multiapi package has only one api-version in per subfolder
-            try:
-                if result[package_name]["isMultiapi"]:
-                    check_api_version_in_subfolder(sdk_code_path)
-            except Exception as e:
-                _LOGGER.warning(
-                    f"Fail to check api version in subfolder for {package_name} in {readme_or_tsp}: {str(e)}"
-                )
-
             # Changelog generation
             try:
                 last_version, last_stable_release = get_version_info(package_name, result[package_name]["tagIsStable"])
@@ -362,7 +229,6 @@ def main(generate_input, generate_output):
                     result[package_name]["tagIsStable"],
                     last_stable_release=last_stable_release,
                     prefolder=folder_name,
-                    is_multiapi=result[package_name]["isMultiapi"],
                 )
 
                 changelog_generation_start_time = time.time()