[SDK generation pipeline] Breaking detector adoption (#36601)

* optimize changelog output

* open breaking change detection for mgmt sdk

* adopt breaking change detector

* format

* update

* update

* update

* update for requirements

* Update package_utils.py

Author: msyyc

scripts/auto_release/main.py

@@ -490,7 +490,7 @@ def create_pr_proc(self):
         pr_body = "" if not self.check_package_size_result else "{}\n".format("\n".join(self.check_package_size_result))
         pr_body = pr_body + "{} \n{} \n{}".format(self.issue_link, self.test_result, self.pipeline_link)
         if not self.is_single_path:
-            pr_body += f'\nBuildTargetingString\n  {self.whole_package_name}\nSkip.CreateApiReview\ntrue'
+            pr_body += f'\nBuildTargetingString\n  {self.whole_package_name}\nSkip.CreateApiReview\ntrue\nSkip.BreakingChanges\ntrue'
         res_create = api.pulls.create(pr_title, pr_head, pr_base, pr_body)
 
         # Add issue link on PR

scripts/auto_release/requirement.txt

@@ -4,4 +4,5 @@ ghapi==0.1.19
 PyGithub==1.55
 packaging==23.1
 pytest==6.2.5
-fastcore==1.3.25
+fastcore==1.3.25
+tox==4.5.0

scripts/breaking_changes_checker/breaking_changes_allowlist.py

@@ -6,7 +6,7 @@
 # --------------------------------------------------------------------------------------------
 
 
-RUN_BREAKING_CHANGES_PACKAGES = []
+RUN_BREAKING_CHANGES_PACKAGES = ["azure-mgmt-*"]
 
 
 # See Readme for ignore format

tools/azure-sdk-tools/packaging_tools/package_utils.py

@@ -1,11 +1,16 @@
+import os
+from typing import Optional
 from pathlib import Path
-
-from subprocess import check_call
+import logging
+from subprocess import check_call, check_output, CalledProcessError
 
 from .change_log import main as change_log_main
 
 DEFAULT_DEST_FOLDER = "./dist"
 
+_LOGGER = logging.getLogger(__name__)
+
+
 # prefolder: "sdk/compute"; name: "azure-mgmt-compute"
 def create_package(prefolder, name, dest_folder=DEFAULT_DEST_FOLDER):
     absdirpath = Path(prefolder, name).absolute()
@@ -16,7 +21,26 @@ def create_package(prefolder, name, dest_folder=DEFAULT_DEST_FOLDER):
     )
 
 
-def change_log_generate(package_name, last_version, tag_is_stable: bool = False):
+def change_log_new(package_folder: str, lastest_pypi_version: bool) -> str:
+    cmd = "tox run -c ../../../eng/tox/tox.ini --root . -e breaking --  --changelog "
+    if lastest_pypi_version:
+        cmd += "--latest-pypi-version"
+    try:
+        output = check_output(cmd, cwd=package_folder, shell=True)
+    except CalledProcessError as e:
+        _LOGGER.warning(f"Failed to generate sdk from typespec: {e.output.decode('utf-8')}")
+        raise e
+    result = [l for l in output.decode("utf-8").split(os.linesep)]
+    begin = result.index("===== changelog start =====")
+    end = result.index("===== changelog end =====")
+    if begin == -1 or end == -1:
+        warn_info = "Failed to get changelog from breaking change detector"
+        _LOGGER.warning(warn_info)
+        raise Exception(warn_info)
+    return "\n".join(result[begin + 1 : end]).strip()
+
+
+def change_log_generate(package_name, last_version, tag_is_stable: bool = False, *, prefolder: Optional[str] = None):
     from pypi_tools.pypi import PyPIClient
 
     client = PyPIClient()
@@ -31,15 +55,27 @@ def change_log_generate(package_name, last_version, tag_is_stable: bool = False)
             last_version[-1] = str(last_release)
     except:
         return "### Other Changes\n\n  - Initial version"
-    else:
-        return change_log_main(f"{package_name}:pypi", f"{package_name}:latest", tag_is_stable)
+
+    # try new changelog tool
+    if prefolder:
+        try:
+            return change_log_new(str(Path(prefolder) / package_name), not (last_stable_release and tag_is_stable))
+        except Exception as e:
+            _LOGGER.warning(f"Failed to generate changelog with breaking_change_detector: {e}")
+
+    # fallback to old changelog tool
+    _LOGGER.info("Fallback to old changelog tool")
+    return change_log_main(f"{package_name}:pypi", f"{package_name}:latest", tag_is_stable)
 
 
 def extract_breaking_change(changelog):
     log = changelog.split("\n")
     breaking_change = []
-    for i in range(0, len(log)):
-        if log[i].find("Breaking Changes") > -1:
-            breaking_change = log[min(i + 2, len(log) - 1) :]
-            break
+    idx = log.index("### Breaking Changes")
+    if idx > -1:
+        for i in range(idx + 1, len(log)):
+            if log[i].find("###") > -1:
+                break
+            if log[i]:
+                breaking_change.append(log[i])
     return sorted([x.replace("  - ", "") for x in breaking_change])

tools/azure-sdk-tools/packaging_tools/sdk_package.py

@@ -24,7 +24,7 @@ def main(generate_input, generate_output):
         last_version = ["first release"]
         if "azure-mgmt-" in package_name:
             try:
-                md_output = change_log_generate(package_name, last_version, package["tagIsStable"])
+                md_output = change_log_generate(package_name, last_version, package["tagIsStable"], prefolder=prefolder)
             except:
                 md_output = "change log generation failed!!!"
         else:
@@ -47,9 +47,19 @@ def main(generate_input, generate_output):
         if "azure-mgmt-" not in package_name:
             try:
                 package_path = Path(sdk_folder, folder_name, package_name)
-                check_call(["python", "-m" "pip", "install", "-r", "../../../eng/apiview_reqs.txt",
-                            "--index-url=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi"
-                            "/simple/"], cwd=package_path, timeout=300)
+                check_call(
+                    [
+                        "python",
+                        "-m" "pip",
+                        "install",
+                        "-r",
+                        "../../../eng/apiview_reqs.txt",
+                        "--index-url=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi"
+                        "/simple/",
+                    ],
+                    cwd=package_path,
+                    timeout=300,
+                )
                 check_call(["apistubgen", "--pkg-path", "."], cwd=package_path, timeout=600)
                 for file in os.listdir(package_path):
                     if "_python.json" in file and package_name in file: