[sdk generation pipeline] simplify generation scripts (#41708)

* Initial plan for issue

* Merge sdk_package functionality into sdk_generator

Co-authored-by: msyyc <[email protected]>

* Fix indentation and improve edge case handling for empty results

Co-authored-by: msyyc <[email protected]>

* Delete sdk_package.py and remove entry point as functionality is now integrated into sdk_generator.py

Co-authored-by: msyyc <[email protected]>

* Optimize flow by integrating packaging directly after generation

Co-authored-by: msyyc <[email protected]>

* remove afterMultiapiCombiner

* update

* update

* update

* update

* update instructions

* Update main.py

---------

Co-authored-by: copilot-swe-agent[bot] <[email protected]>

Author: msyyc

scripts/auto_release/main.py

@@ -89,7 +89,6 @@ def __init__(self):
         self.pr_number = 0
         self.tag_is_stable = False
         self.has_test = False
-        self.check_package_size_result = []  # List[str]
         self.version_suggestion = ""  # if can't calculate next version, give a suggestion
 
     @property
@@ -165,11 +164,9 @@ def generate_code(self):
         print_check(f"python -m packaging_tools.sdk_generator {self.autorest_result} {self.autorest_result}")
 
         generate_result = self.get_autorest_result()
-        self.tag_is_stable = list(generate_result.values())[0]["tagIsStable"]
+        self.tag_is_stable = generate_result["packages"][0]["tagIsStable"]
         log(f"tag_is_stable is {self.tag_is_stable}")
 
-        print_check(f"python -m packaging_tools.sdk_package {self.autorest_result} {self.autorest_result}")
-
     def get_package_name_with_autorest_result(self):
         generate_result = self.get_autorest_result()
         self.whole_package_name = generate_result["packages"][0]["packageName"]
@@ -210,15 +207,6 @@ def get_last_release_version(self) -> str:
         except:
             return ""
 
-    def check_package_size(self):
-        if self.after_multiapi_combiner:
-            packages = self.get_private_package()
-            for package in packages:
-                if os.path.getsize(package) > 2 * 1024 * 1024:
-                    self.check_package_size_result.append(
-                        f"ERROR: Package size is over 2MBytes: {Path(package).name}!!!"
-                    )
-
     def check_model_flatten(self):
         if self.whole_package_name in [
             "azure-mgmt-mysqlflexibleservers",
@@ -256,7 +244,6 @@ def check_model_flatten(self):
                     raise Exception(message)
 
     def check_file(self):
-        self.check_package_size()
         self.check_model_flatten()
 
     def sdk_code_path(self) -> str:
@@ -333,8 +320,7 @@ def create_pr_proc(self):
         pr_title = "[AutoRelease] {}(can only be merged by SDK owner)".format(self.new_branch)
         pr_head = "{}:{}".format(os.getenv("USR_NAME"), self.new_branch)
         pr_base = "main"
-        pr_body = "" if not self.check_package_size_result else "{}\n".format("\n".join(self.check_package_size_result))
-        pr_body = pr_body + "{} \n{} \n{}".format(self.issue_link, self.test_result, self.pipeline_link)
+        pr_body = "{} \n{} \n{}".format(self.issue_link, self.test_result, self.pipeline_link)
         if self.has_multi_packages:
             pr_body += f"\nBuildTargetingString\n  {self.whole_package_name}\nSkip.CreateApiReview"
         res_create = api.pulls.create(pr_title, pr_head, pr_base, pr_body)
@@ -351,15 +337,6 @@ def zero_version_policy(self):
                 issue_number = int(self.issue_link.split("/")[-1])
                 api_request.issues.add_labels(issue_number=issue_number, labels=["base-branch-attention"])
 
-    @property
-    def after_multiapi_combiner(self) -> bool:
-        content = self.get_autorest_result()
-        return content["packages"][0]["afterMultiapiCombiner"]
-
-    def get_private_package(self) -> List[str]:
-        content = self.get_autorest_result()
-        return content["packages"][0]["artifacts"]
-
     def ask_check_policy(self):
         changelog = self.get_changelog()
         if changelog == "":

scripts/automation_generate.sh

@@ -1,22 +1,10 @@
 #!/bin/bash
 
 TEMP_FILE="$TMPDIR/auto_temp.json"
-# generate code
-python -m packaging_tools.sdk_generator "$1" "$TEMP_FILE" --debug 2>&1
-echo "[Generate] codegen done!!!"
-if [ ! -f "$TEMP_FILE" ]; then
-  echo "[Autorest]$TEMP_FILE does not exist!!!Error happened during codegen"
-  exit 1
-fi
-
-if [ -f "$2" ]; then
-  rm "$2"
-fi
-
-# package
-python -m packaging_tools.sdk_package "$TEMP_FILE" "$2" --debug 2>&1
-echo "[Generate] generate done!!!"
+# generate code and package in one step
+python -m packaging_tools.sdk_generator "$1" "$2" --debug 2>&1
+echo "[Generate] generation and packaging done!!!"
 if [ ! -f "$2" ]; then
-  echo "[Autorest]$2 does not exist!!!Error happened during package"
+  echo "[Autorest]$2 does not exist!!!Error happened during generation"
   exit 1
 fi

scripts/sdk_generate.sh

@@ -15,18 +15,10 @@ echo "$PATH"
 export PATH="/usr/local/n/versions/node/18.19.0/bin:$PATH"
 
 TEMP_FILE="$TMPDIR/venv-sdk/auto_temp.json"
-# generate code
-python -m packaging_tools.sdk_generator "$1" "$TEMP_FILE" 2>&1
-echo "[Generate] codegen done!!!"
-if [ ! -f "$TEMP_FILE" ]; then
-  echo "[Autorest]$TEMP_FILE does not exist!!!Error happened during codegen"
-  exit 1
-fi
-
-# package
-python -m packaging_tools.sdk_package "$TEMP_FILE" "$2" 2>&1
-echo "[Generate] generate done!!!"
+# generate code and package in one step
+python -m packaging_tools.sdk_generator "$1" "$2" 2>&1
+echo "[Generate] generation and packaging done!!!"
 if [ ! -f "$2" ]; then
-  echo "[Autorest]$2 does not exist!!!Error happened during package"
+  echo "[Autorest]$2 does not exist!!!Error happened during generation"
   exit 1
 fi

tools/azure-sdk-tools/packaging_tools/sdk_generator.py

@@ -10,6 +10,7 @@
 import re
 import os
 from functools import partial
+import multiprocessing
 
 try:
     # py 311 adds this library natively
@@ -38,6 +39,7 @@
     del_outdated_generated_files,
 )
 from .conf import CONF_NAME
+from .package_utils import create_package, change_log_generate, extract_breaking_change, get_version_info, check_file
 
 logging.basicConfig(
     stream=sys.stdout,
@@ -47,6 +49,11 @@
 _LOGGER = logging.getLogger(__name__)
 
 
+def execute_func_with_timeout(func, timeout: int = 900) -> Any:
+    """Execute function with timeout"""
+    return multiprocessing.Pool(processes=1).apply_async(func).get(timeout)
+
+
 def is_multiapi_package(python_md_content: List[str]) -> bool:
     for line in python_md_content:
         if re.findall(r"\s*multiapi\s*:\s*true", line):
@@ -258,7 +265,7 @@ def main(generate_input, generate_output):
                 config = gen_dpg(readme_or_tsp, data.get("autorestConfig", ""), dpg_relative_folder(spec_folder))
             _LOGGER.info(f"code generation cost time: {int(time.time() - code_generation_start_time)} seconds")
         except Exception as e:
-            _LOGGER.error(f"fail to generate sdk for {readme_or_tsp}: {str(e)}")
+            _LOGGER.error(f"Fail to generate sdk for {readme_or_tsp}: {str(e)}")
             for hint_message in [
                 "======================================= Whant Can I do (begin) ========================================================================",
                 f"Fail to generate sdk for {readme_or_tsp}. If you are from service team, please first check if the failure happens only to Python automation, or for all SDK automations. ",
@@ -293,43 +300,157 @@ def main(generate_input, generate_output):
                     package_entry["isMultiapi"] = is_multiapi_package(readme_python_content)
                     package_entry["targetReleaseDate"] = data.get("targetReleaseDate", "")
                     package_entry["allowInvalidNextVersion"] = data.get("allowInvalidNextVersion", False)
-                    package_entry["runInPipeline"] = run_in_pipeline
                     result[package_name] = package_entry
                 else:
                     result[package_name]["path"].append(folder_name)
                     result[package_name][spec_word].append(readme_or_tsp)
+            except Exception as e:
+                _LOGGER.error(f"Fail to process package {package_name} in {readme_or_tsp}: {str(e)}")
+                continue
 
-                # Generate some necessary file for new service
+            # Generate some necessary file for new service
+            try:
                 init_new_service(package_name, folder_name)
+            except Exception as e:
+                _LOGGER.warning(f"Fail to init new service {package_name} in {readme_or_tsp}: {str(e)}")
+
+            # format samples and tests
+            try:
                 format_samples_and_tests(sdk_code_path)
+            except Exception as e:
+                _LOGGER.warning(f"Fail to format samples and tests for {package_name} in {readme_or_tsp}: {str(e)}")
 
-                # Update metadata
-                try:
-                    update_servicemetadata(
-                        sdk_folder,
-                        data,
-                        config,
-                        folder_name,
-                        package_name,
-                        spec_folder,
-                        readme_or_tsp,
-                    )
-                except Exception as e:
-                    _LOGGER.error(f"fail to update meta: {str(e)}")
+            # Update metadata
+            try:
+                update_servicemetadata(
+                    sdk_folder,
+                    data,
+                    config,
+                    folder_name,
+                    package_name,
+                    spec_folder,
+                    readme_or_tsp,
+                )
+            except Exception as e:
+                _LOGGER.warning(f"Fail to update meta: {str(e)}")
 
-                # Setup package locally
+            # Setup package locally
+            try:
                 check_call(
                     f"pip install --ignore-requires-python -e {sdk_code_path}",
                     shell=True,
                 )
+            except Exception as e:
+                _LOGGER.warning(f"Fail to setup package {package_name} in {readme_or_tsp}: {str(e)}")
+
+            # check whether multiapi package has only one api-version in per subfolder
+            try:
+                if result[package_name]["isMultiapi"]:
+                    check_api_version_in_subfolder(sdk_code_path)
+            except Exception as e:
+                _LOGGER.warning(
+                    f"Fail to check api version in subfolder for {package_name} in {readme_or_tsp}: {str(e)}"
+                )
+
+            # Changelog generation
+            try:
+                last_version, last_stable_release = get_version_info(package_name, result[package_name]["tagIsStable"])
+                change_log_func = partial(
+                    change_log_generate,
+                    package_name,
+                    last_version,
+                    result[package_name]["tagIsStable"],
+                    last_stable_release=last_stable_release,
+                    prefolder=folder_name,
+                    is_multiapi=result[package_name]["isMultiapi"],
+                )
+
+                changelog_generation_start_time = time.time()
+                try:
+                    md_output = execute_func_with_timeout(change_log_func)
+                except multiprocessing.TimeoutError:
+                    md_output = "change log generation was timeout!!! You need to write it manually!!!"
+                except:
+                    md_output = "change log generation failed!!! You need to write it manually!!!"
+                finally:
+                    for file in ["stable.json", "current.json"]:
+                        file_path = Path(sdk_folder, folder_name, package_name, file)
+                        if file_path.exists():
+                            os.remove(file_path)
+                            _LOGGER.info(f"Remove {file_path} which is temp file to generate changelog.")
+
+                _LOGGER.info(
+                    f"changelog generation cost time: {int(time.time() - changelog_generation_start_time)} seconds"
+                )
+                result[package_name]["changelog"] = {
+                    "content": md_output,
+                    "hasBreakingChange": "Breaking Changes" in md_output,
+                    "breakingChangeItems": extract_breaking_change(md_output),
+                }
+                result[package_name]["version"] = last_version
+
+                _LOGGER.info(f"[PACKAGE]({package_name})[CHANGELOG]:{md_output}")
+            except Exception as e:
+                _LOGGER.warning(f"Fail to generate changelog for {package_name} in {readme_or_tsp}: {str(e)}")
+
+            # Generate ApiView
+            if run_in_pipeline:
+                apiview_start_time = time.time()
+                try:
+                    package_path = Path(sdk_folder, folder_name, package_name)
+                    check_call(
+                        [
+                            "python",
+                            "-m",
+                            "pip",
+                            "install",
+                            "-r",
+                            "../../../eng/apiview_reqs.txt",
+                            "--index-url=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi"
+                            "/simple/",
+                        ],
+                        cwd=package_path,
+                        timeout=600,
+                    )
+                    check_call(["apistubgen", "--pkg-path", "."], cwd=package_path, timeout=600)
+                    for file in os.listdir(package_path):
+                        if "_python.json" in file and package_name in file:
+                            result[package_name]["apiViewArtifact"] = str(Path(package_path, file))
+                except Exception as e:
+                    _LOGGER.debug(f"Fail to generate ApiView token file for {package_name}: {e}")
+                _LOGGER.info(f"apiview generation cost time: {int(time.time() - apiview_start_time)} seconds")
+            else:
+                _LOGGER.info("Skip ApiView generation for package that does not run in pipeline.")
 
-                # check whether multiapi package has only one api-version in per subfolder
-                check_api_version_in_subfolder(sdk_code_path)
+            # check generated files and update package["version"]
+            if package_name.startswith("azure-mgmt-"):
+                try:
+                    check_file(result[package_name])
+                except Exception as e:
+                    _LOGGER.warning(f"Fail to check generated files for {package_name}: {e}")
 
-                # could be removed in the short future
-                result[package_name]["afterMultiapiCombiner"] = False
+            # Build artifacts for package
+            try:
+                create_package(result[package_name]["path"][0], package_name)
+                dist_path = Path(sdk_folder, folder_name, package_name, "dist")
+                result[package_name]["artifacts"] = [
+                    str(dist_path / package_file) for package_file in os.listdir(dist_path)
+                ]
+                for artifact in result[package_name]["artifacts"]:
+                    if ".whl" in artifact:
+                        result[package_name]["language"] = "Python"
+                        break
+                _LOGGER.info(f"Built package {package_name} successfully.")
             except Exception as e:
-                _LOGGER.error(f"fail to setup package: {str(e)}")
+                _LOGGER.warning(f"Fail to build package {package_name} in {readme_or_tsp}: {str(e)}")
+
+            # update result
+            result[package_name]["installInstructions"] = {
+                "full": "You can use pip to install the artifacts.",
+                "lite": f"pip install {package_name}",
+            }
+            result[package_name]["result"] = "succeeded"
+            result[package_name]["packageFolder"] = result[package_name]["path"][0]
 
     # remove duplicates
     try:
@@ -340,13 +461,23 @@ def main(generate_input, generate_output):
             if value.get("readmeMd"):
                 value["readmeMd"] = list(set(value["readmeMd"]))
     except Exception as e:
-        _LOGGER.error(f"fail to remove duplicates: {str(e)}")
+        _LOGGER.warning(f"Fail to remove duplicates: {str(e)}")
 
     if len(result) == 0 and len(readme_and_tsp) > 1:
         raise Exception("No package is generated, please check the log for details")
 
+    if len(result) == 0:
+        _LOGGER.info("No packages to process, returning empty result")
+    else:
+        _LOGGER.info(f"Processing {len(result)} generated packages...")
+
+    final_result = {"packages": list(result.values())}
     with open(generate_output, "w") as writer:
-        json.dump(result, writer)
+        json.dump(final_result, writer, indent=2)
+
+    _LOGGER.info(
+        f"Congratulations! Succeed to build package for {[p['packageName'] for p in final_result['packages']]}. And you shall be able to see the generated code when running 'git status'."
+    )
 
 
 def generate_main():