[Pipeline] optimize for batch run in pipeline (Azure#40213)

* optimize for batch run

* update

* update

* add log

* fix

Author: msyyc

scripts/automation_init.sh

@@ -9,7 +9,7 @@ pip install setuptools==78.1.0 > /dev/null
 
 # install tsp-client globally (local install may interfere with tooling)
 echo Install tsp-client
-sudo npm install -g @azure-tools/typespec-client-generator-cli > /dev/null
+sudo npm install -g @azure-tools/typespec-client-generator-cli@0.20.0 > /dev/null
 
 echo "{}" >> $2
 echo "[Generate] init success!!!"

tools/azure-sdk-tools/packaging_tools/generate_utils.py

@@ -416,13 +416,31 @@ def generate_ci(template_path: Path, folder_path: Path, package_name: str) -> No
         file_out.writelines(content)
 
 
-def gen_typespec(typespec_relative_path: str, spec_folder: str, head_sha: str, rest_repo_url: str) -> Dict[str, Any]:
+def gen_typespec(
+    typespec_relative_path: str,
+    spec_folder: str,
+    head_sha: str,
+    rest_repo_url: str,
+    run_in_pipeline: bool,
+) -> Dict[str, Any]:
     typespec_python = "@azure-tools/typespec-python"
     # call scirpt to generate sdk
     try:
         tsp_dir = (Path(spec_folder) / typespec_relative_path).resolve()
         repo_url = rest_repo_url.replace("https://github.com/", "")
-        cmd = f"tsp-client init --tsp-config {tsp_dir} --local-spec-repo {tsp_dir} --commit {head_sha} --repo {repo_url} --debug"
+        cmd = (
+            f"tsp-client init --tsp-config {tsp_dir} --local-spec-repo {tsp_dir} --commit {head_sha} --repo {repo_url}"
+        )
+        if run_in_pipeline:
+            emitter_name = "@azure-tools/typespec-python"
+            if not os.path.exists(f"node_modules/{emitter_name}"):
+                _LOGGER.info("install dependencies only for the first run")
+                check_output("tsp-client install-dependencies", stderr=STDOUT, shell=True)
+            else:
+                _LOGGER.info(f"skip install since {emitter_name} is already installed")
+            cmd += " --skip-install --debug"
+        else:
+            cmd += " --debug"
         _LOGGER.info(f"generation cmd: {cmd}")
         output = check_output(cmd, stderr=STDOUT, shell=True)
     except CalledProcessError as e:

tools/azure-sdk-tools/packaging_tools/sdk_generator.py

@@ -221,6 +221,7 @@ def main(generate_input, generate_output):
     python_tag = data.get("python_tag")
     package_total = set()
     readme_and_tsp = data.get("relatedReadmeMdFiles", []) + data.get("relatedTypeSpecProjectFolder", [])
+    run_in_pipeline = data.get("runMode") is not None
     for readme_or_tsp in readme_and_tsp:
         _LOGGER.info(f"[CODEGEN]({readme_or_tsp})codegen begin")
         try:
@@ -245,7 +246,7 @@ def main(generate_input, generate_output):
                 config = gen_dpg(readme_or_tsp, data.get("autorestConfig", ""), dpg_relative_folder(spec_folder))
             else:
                 del_outdated_generated_files(str(Path(spec_folder, readme_or_tsp)))
-                config = gen_typespec(readme_or_tsp, spec_folder, data["headSha"], data["repoHttpsUrl"])
+                config = gen_typespec(readme_or_tsp, spec_folder, data["headSha"], data["repoHttpsUrl"], run_in_pipeline)
             _LOGGER.info(f"code generation cost time: {int(time.time() - code_generation_start_time)} seconds")
         except Exception as e:
             _LOGGER.error(f"fail to generate sdk for {readme_or_tsp}: {str(e)}")