[sdk generation pipeline] merge "sdk_packaging.toml" into "pyproject.toml" (#42300)

* replace sdk_packaging.toml with pyproject.toml

* update

* update

* update

* update

* update

* update

* Update tools/azure-sdk-tools/packaging_tools/package_utils.py

Co-authored-by: Copilot <[email protected]>

---------

Co-authored-by: Copilot <[email protected]>

Author: msyyc

scripts/auto_release/requirement.txt

@@ -8,4 +8,6 @@ fastcore==1.3.25
 tox==4.15.0
 wheel==0.43.0
 setuptools==78.1.0
-build==1.0.3
+build==1.0.3
+tomli-w==1.0.0
+tomli==2.2.1

tools/azure-sdk-tools/packaging_tools/conf.py

@@ -12,7 +12,8 @@
 
 _LOGGER = logging.getLogger(__name__)
 
-CONF_NAME = "sdk_packaging.toml"
+CONF_NAME = "pyproject.toml"
+OLD_CONF_NAME = "sdk_packaging.toml"
 _SECTION = "packaging"
 
 # Default conf

tools/azure-sdk-tools/packaging_tools/generate_utils.py

@@ -2,6 +2,13 @@
 import logging
 import os
 import re
+try:
+    # py 311 adds this library natively
+    import tomllib as toml
+except:
+    # otherwise fall back to pypi package tomli
+    import tomli as toml
+import tomli_w as tomlw
 from functools import wraps
 from typing import Optional
 
@@ -16,7 +23,7 @@
 from . import build_packaging
 from .swaggertosdk.autorest_tools import build_autorest_options, generate_code
 from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE_DPG, read_config
-from .conf import CONF_NAME
+from .conf import CONF_NAME, OLD_CONF_NAME
 from jinja2 import Environment, FileSystemLoader
 
 
@@ -113,11 +120,43 @@ def call_build_config(package_name: str, folder_name: str):
     # )
 
 
-def init_new_service(package_name, folder_name):
+def generate_packaging_files(package_name, folder_name):
+    # replace sdk_packaging.toml with pyproject.toml
+    output_path = Path(folder_name) / package_name
+    pyproject_toml = output_path / CONF_NAME
+    sdk_packaging_toml = output_path / OLD_CONF_NAME
+    if sdk_packaging_toml.exists():
+        if pyproject_toml.exists():
+            # update related items in pyproject.toml then delete sdk_packaging.toml
+            _LOGGER.info(f"update {pyproject_toml} with {sdk_packaging_toml}")
+
+            # Read the old sdk_packaging.toml content
+            with open(sdk_packaging_toml, "rb") as f:
+                sdk_packaging_content = toml.load(f)
+
+            # Read the existing pyproject.toml content
+            with open(pyproject_toml, "rb") as f:
+                pyproject_content = toml.load(f)
+
+            # Update pyproject.toml with sdk_packaging.toml content
+            pyproject_content.update(sdk_packaging_content)
+
+            # Write updated content back to pyproject.toml
+            with open(pyproject_toml, "wb") as f:
+                tomlw.dump(pyproject_content, f)
+
+            # Delete the old sdk_packaging.toml file
+            sdk_packaging_toml.unlink()
+            _LOGGER.info(f"deleted {sdk_packaging_toml}")
+
+        else:
+            # rename sdk_packaging.toml to pyproject.toml
+            _LOGGER.info(f"rename {sdk_packaging_toml} to {pyproject_toml}")
+            sdk_packaging_toml.rename(pyproject_toml)
+
     if "azure-mgmt-" in package_name:
         call_build_config(package_name, folder_name)
     else:
-        output_path = Path(folder_name) / package_name
         if not (output_path / CONF_NAME).exists():
             with open(output_path / CONF_NAME, "w") as file_out:
                 file_out.write("[packaging]\nauto_update = false")

tools/azure-sdk-tools/packaging_tools/package_utils.py

@@ -2,6 +2,14 @@
 import ast
 import time
 import shutil
+
+try:
+    # py 311 adds this library natively
+    import tomllib as toml
+except:
+    # otherwise fall back to pypi package tomli
+    import tomli as toml
+import tomli_w as tomlw
 from typing import Optional, Tuple, Dict, Any, List
 from pathlib import Path
 import logging
@@ -223,28 +231,21 @@ def check_file_with_packaging_tool(self):
         if not title:
             _LOGGER.info(f"Can not find the title for {self.whole_package_name}")
 
-        # add `title` and update `is_stable` in sdk_packaging.toml
-        toml = Path(self.whole_package_name) / "sdk_packaging.toml"
-        stable_config = f'is_stable = {"true" if self.tag_is_stable and self.next_version != "1.0.0b1" else "false"}\n'
-        if toml.exists():
-
-            def edit_toml(content: List[str]):
-                has_title = False
-                has_isstable = False
-                for idx in range(len(content)):
-                    if "title" in content[idx]:
-                        has_title = True
-                    if "is_stable" in content[idx]:
-                        has_isstable = True
-                        content[idx] = stable_config
-                if not has_title:
-                    content.append(f'title = "{title}"\n')
-                if not has_isstable:
-                    content.append(stable_config)
-
-            modify_file(str(toml), edit_toml)
+        # add `title` and update `is_stable` in pyproject.toml
+        pyproject_toml = Path(self.whole_package_name) / "pyproject.toml"
+        if pyproject_toml.exists():
+            with open(pyproject_toml, "rb") as fd:
+                toml_data = toml.load(fd)
+            if "packaging" not in toml_data:
+                toml_data["packaging"] = {}
+            if title and not toml_data["packaging"].get("title"):
+                toml_data["packaging"]["title"] = title
+            toml_data["packaging"]["is_stable"] = self.tag_is_stable and self.next_version != "1.0.0b1"
+            with open(pyproject_toml, "wb") as fd:
+                tomlw.dump(toml_data, fd)
+            _LOGGER.info(f"Update {pyproject_toml} successfully")
         else:
-            _LOGGER.info(f"{os.getcwd()}/{toml} does not exist")
+            _LOGGER.info(f"{os.getcwd()}/{pyproject_toml} does not exist")
 
         build_packaging(output_folder=".", packages=[self.whole_package_name], build_conf=True)
         _LOGGER.info("packaging_tools --build-conf successfully")
@@ -381,42 +382,44 @@ def check_pyproject_toml(self):
             "mypy": "false",
         }
 
-        # Create new pyproject.toml if it doesn't exist
-        if not toml_path.exists():
-            with open(toml_path, "w") as file:
-                file.write("[tool.azure-sdk-build]\n")
-                for key, value in default_configs.items():
-                    file.write(f"{key} = {value}\n")
-                _LOGGER.info("Created pyproject.toml with default configurations")
-            return
-
-        # If file exists, ensure all required configurations are present
-        def edit_toml(content: List[str]):
-            # Track if we have the [tool.azure-sdk-build] section
-            has_section = False
-            config_exists = {key: False for key in default_configs}
-
-            # Check for existing configurations and section
-            for i, line in enumerate(content):
-                if "[tool.azure-sdk-build]" in line:
-                    has_section = True
-
-                # Check for each configuration
-                for key in default_configs:
-                    if f"{key} = " in line:
-                        config_exists[key] = True
-
-            # Add section if it doesn't exist
-            if not has_section:
-                content.append("\n[tool.azure-sdk-build]\n")
-
-            # Add missing configurations
-            for key, value in default_configs.items():
-                if not config_exists[key]:
-                    _LOGGER.info(f"Adding {key} = {value} to pyproject.toml")
-                    content.append(f"{key} = {value}\n")
-
-        modify_file(str(toml_path), edit_toml)
+        # Load existing TOML or create new structure
+        if toml_path.exists():
+            try:
+                with open(toml_path, "rb") as file:
+                    toml_data = toml.load(file)
+            except Exception as e:
+                _LOGGER.warning(f"Error parsing pyproject.toml: {e}, creating new one")
+                toml_data = {}
+        else:
+            toml_data = {}
+
+        # Ensure [tool.azure-sdk-build] section exists
+        if "tool" not in toml_data:
+            toml_data["tool"] = {}
+
+        if "azure-sdk-build" not in toml_data["tool"]:
+            toml_data["tool"]["azure-sdk-build"] = {}
+
+        # Update configurations
+        azure_sdk_build = toml_data["tool"]["azure-sdk-build"]
+
+        for key, value in default_configs.items():
+            if key not in azure_sdk_build:
+                _LOGGER.info(f"Adding {key} = {value} to pyproject.toml")
+                if isinstance(value, str):
+                    if value.lower() == "true":
+                        azure_sdk_build[key] = True
+                    elif value.lower() == "false":
+                        azure_sdk_build[key] = False
+                    else:
+                        azure_sdk_build[key] = value
+                else:
+                    azure_sdk_build[key] = value
+        # Write back to file
+        with open(toml_path, "wb") as file:
+            tomlw.dump(toml_data, file)
+
+        _LOGGER.info("Updated pyproject.toml with required azure-sdk-build configurations")
 
     def run(self):
         self.check_file_with_packaging_tool()

tools/azure-sdk-tools/packaging_tools/sdk_generator.py

@@ -26,7 +26,7 @@
 from .generate_sdk import generate
 from .generate_utils import (
     get_package_names,
-    init_new_service,
+    generate_packaging_files,
     update_servicemetadata,
     judge_tag_preview,
     format_samples_and_tests,
@@ -309,11 +309,11 @@ def main(generate_input, generate_output):
                 _LOGGER.error(f"Fail to process package {package_name} in {readme_or_tsp}: {str(e)}")
                 continue
 
-            # Generate some necessary file for new service
+            # Generate packaging files
             try:
-                init_new_service(package_name, folder_name)
+                generate_packaging_files(package_name, folder_name)
             except Exception as e:
-                _LOGGER.warning(f"Fail to init new service {package_name} in {readme_or_tsp}: {str(e)}")
+                _LOGGER.warning(f"Fail to generate packaging files for {package_name} in {readme_or_tsp}: {str(e)}")
 
             # format samples and tests
             try: