[SDK generation pipeline] optimize log (#36720)

* optimize log

* add hint message

* update stderr

* update log format

* update log format

* update hint message

* update

* update

Author: msyyc

swagger_to_sdk_config.json

@@ -6,17 +6,17 @@
   },
   "initOptions": {
     "initScript": {
-      "path": "sh scripts/automation_init.sh"
+      "path": "sh scripts/automation_init.sh",
+      "stderr": {
+        "showInComment": false
+      }
     }
   },
   "generateOptions": {
     "generateScript": {
       "path": "sh scripts/automation_generate.sh",
-      "stderr": {
-        "showInComment": true
-      },
       "stdout": {
-        "showInComment": "^\\[Autorest\\]"
+        "showInComment": "(.*\\[ERROR\\]\\s*).*"
       }
     },
     "parseGenerateOutput": true

tools/azure-sdk-tools/packaging_tools/generate_sdk.py

@@ -1,3 +1,4 @@
+import sys
 import argparse
 import logging
 from pathlib import Path
@@ -15,6 +16,11 @@
     get_repo_tag_meta,
 )
 
+logging.basicConfig(
+    stream=sys.stdout,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    datefmt="%Y-%m-%d %X",
+)
 _LOGGER = logging.getLogger(__name__)
 
 

tools/azure-sdk-tools/packaging_tools/generate_utils.py

@@ -7,7 +7,7 @@
 
 from ci_tools.git_tools import get_add_diff_file_list
 from pathlib import Path
-from subprocess import check_output, CalledProcessError, check_call, getoutput
+from subprocess import check_output, CalledProcessError, check_call
 from typing import Dict, Any
 from glob import glob
 import yaml

tools/azure-sdk-tools/packaging_tools/package_utils.py

@@ -71,7 +71,11 @@ def change_log_generate(package_name, last_version, tag_is_stable: bool = False,
 def extract_breaking_change(changelog):
     log = changelog.split("\n")
     breaking_change = []
-    idx = log.index("### Breaking Changes")
+    idx = -1
+    try:
+        idx = log.index("### Breaking Changes")
+    except ValueError:
+        pass
     if idx > -1:
         for i in range(idx + 1, len(log)):
             if log[i].find("###") > -1:

tools/azure-sdk-tools/packaging_tools/sdk_generator.py

@@ -1,3 +1,4 @@
+import sys
 from typing import List, Dict, Any
 import argparse
 import json
@@ -35,6 +36,11 @@
 )
 from .conf import CONF_NAME
 
+logging.basicConfig(
+    stream=sys.stdout,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    datefmt="%Y-%m-%d %X",
+)
 _LOGGER = logging.getLogger(__name__)
 
 
@@ -275,25 +281,37 @@ def main(generate_input, generate_output):
 
     for input_readme in readme_files:
         _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin")
-        if "resource-manager" in input_readme:
-            relative_path_readme = str(Path(spec_folder, input_readme))
-            update_metadata_for_multiapi_package(spec_folder, input_readme)
-            del_outdated_files(relative_path_readme)
-            config = generate(
-                CONFIG_FILE,
-                sdk_folder,
-                [],
-                relative_path_readme,
-                spec_folder,
-                force_generation=True,
-                python_tag=python_tag,
-            )
-        elif "data-plane" in input_readme:
-            config = gen_dpg(input_readme, data.get("autorestConfig", ""), dpg_relative_folder(spec_folder))
-        else:
-            del_outdated_generated_files(str(Path(spec_folder, input_readme)))
-            config = gen_typespec(input_readme, spec_folder, data["headSha"], data["repoHttpsUrl"])
-        package_names = get_package_names(sdk_folder)
+        try:
+            if "resource-manager" in input_readme:
+                relative_path_readme = str(Path(spec_folder, input_readme))
+                update_metadata_for_multiapi_package(spec_folder, input_readme)
+                del_outdated_files(relative_path_readme)
+                config = generate(
+                    CONFIG_FILE,
+                    sdk_folder,
+                    [],
+                    relative_path_readme,
+                    spec_folder,
+                    force_generation=True,
+                    python_tag=python_tag,
+                )
+            elif "data-plane" in input_readme:
+                config = gen_dpg(input_readme, data.get("autorestConfig", ""), dpg_relative_folder(spec_folder))
+            else:
+                del_outdated_generated_files(str(Path(spec_folder, input_readme)))
+                config = gen_typespec(input_readme, spec_folder, data["headSha"], data["repoHttpsUrl"])
+            package_names = get_package_names(sdk_folder)
+        except Exception as e:
+            _LOGGER.error(f"fail to generate sdk for {input_readme}: {str(e)}")
+            for hint_message in [
+                "======================================= Whant Can I do ========================================================================",
+                "If you are from service team, please first check if the failure happens only to Python automation, or for all SDK automations. ",
+                "If it happens for all SDK automations, please double check your Swagger / Typespec, and check whether there is error in ModelValidation and LintDiff. ",
+                "If it happens to Python alone, you can open an issue to https://github.com/Azure/autorest.python/issues. Please include the link of this Pull Request in the issue.",
+                "===============================================================================================================================",
+            ]:
+                _LOGGER.error(hint_message)
+            raise e
         _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]")
 
         # folder_name: "sdk/containerservice"; package_name: "azure-mgmt-containerservice"
@@ -330,7 +348,7 @@ def main(generate_input, generate_output):
                     input_readme,
                 )
             except Exception as e:
-                _LOGGER.info(f"fail to update meta: {str(e)}")
+                _LOGGER.error(f"fail to update meta: {str(e)}")
 
             # Setup package locally
             check_call(

tools/azure-sdk-tools/packaging_tools/sdk_package.py

@@ -1,3 +1,4 @@
+import sys
 import argparse
 import json
 import logging
@@ -7,6 +8,11 @@
 
 from .package_utils import create_package, change_log_generate, extract_breaking_change
 
+logging.basicConfig(
+    stream=sys.stdout,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    datefmt="%Y-%m-%d %X",
+)
 _LOGGER = logging.getLogger(__name__)
 
 

tools/azure-sdk-tools/packaging_tools/swaggertosdk/autorest_tools.py

@@ -1,10 +1,16 @@
+import sys
 import json
 import logging
 import os.path
 from pathlib import Path
 import shutil
 import subprocess
 
+logging.basicConfig(
+    stream=sys.stdout,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    datefmt="%Y-%m-%d %X",
+)
 _LOGGER = logging.getLogger(__name__)
 
 
@@ -145,7 +151,7 @@ def run_command():
         if process.returncode:
             # print necessary error info which will be displayed in swagger pr
             for i in range(-min(len(output_buffer), 20), 0):
-                print(f"[Autorest] {output_buffer[i]}")
+                _LOGGER.error(output_buffer[i])
             raise subprocess.CalledProcessError(process.returncode, cmd_line, output)
         return output