Enable package mode for mgmt sdk (#42276)

* enable package mode for mgmt sdk

* update classifier logic

* fix ofr __init__.py

* Update __init__.py

* update

* update

* fix

* Update swagger_to_sdk_config_autorest.json

* update

* fix for new service

* fix for build

* update

* fix for judge tag

Author: msyyc

eng/tools/azure-sdk-tools/packaging_tools/__init__.py

@@ -51,6 +51,8 @@ def build_packaging(
     jenkins: bool = False,
     packages: List[str] = None,
     build_conf: bool = False,
+    *,
+    template_names: Optional[List[str]] = None,
 ) -> None:
     package_names = set(packages) or set()
     if jenkins:
@@ -69,10 +71,12 @@ def build_packaging(
         raise ValueError("Was unable to find out the package names.")
 
     for package_name in package_names:
-        build_packaging_by_package_name(package_name, output_folder, build_conf)
+        build_packaging_by_package_name(package_name, output_folder, build_conf, template_names)
 
 
-def build_packaging_by_package_name(package_name: str, output_folder: str, build_conf: bool = False) -> None:
+def build_packaging_by_package_name(
+    package_name: str, output_folder: str, build_conf: bool = False, template_names: Optional[List[str]] = None
+) -> None:
     _LOGGER.info("Building template %s", package_name)
     package_folder = Path(output_folder) / Path(package_name)
 
@@ -90,7 +94,11 @@ def build_packaging_by_package_name(package_name: str, output_folder: str, build
     env = Environment(loader=PackageLoader("packaging_tools", "templates/packaging_files"), keep_trailing_newline=True)
     conf = build_config(conf)
 
+    template_names = template_names or env.list_templates()
     for template_name in env.list_templates():
+        if template_name not in template_names:
+            _LOGGER.info("Skipping template %s", template_name)
+            continue
         future_filepath = Path(output_folder) / package_name / template_name
 
         # Might decide to make it more generic one day

eng/tools/azure-sdk-tools/packaging_tools/conf.py

@@ -42,14 +42,19 @@ def read_conf(folder: Path) -> Dict[str, Any]:
 
 def build_default_conf(folder: Path, package_name: str) -> None:
     conf_path = folder / CONF_NAME
+    existing_conf = {_SECTION: {}}
     if conf_path.exists():
-        _LOGGER.info("Skipping default conf since the file exists")
-        return
+        with open(conf_path, "rb") as fd:
+            existing_conf = toml.load(fd)
+        if _SECTION not in existing_conf:
+            existing_conf[_SECTION] = {}
 
     _LOGGER.info("Build default conf for %s", package_name)
-    conf = {_SECTION: _CONFIG.copy()}
-    conf[_SECTION]["package_name"] = package_name
-    conf[_SECTION]["package_nspkg"] = package_name[: package_name.rindex("-")] + "-nspkg"
+    for k in _CONFIG:
+        if k not in existing_conf[_SECTION]:
+            existing_conf[_SECTION][k] = _CONFIG[k]
+    existing_conf[_SECTION]["package_name"] = package_name
+    existing_conf[_SECTION]["package_nspkg"] = package_name[: package_name.rindex("-")] + "-nspkg"
 
     with open(conf_path, "wb") as fd:
-        tomlw.dump(conf, fd)
+        tomlw.dump(existing_conf, fd)

eng/tools/azure-sdk-tools/packaging_tools/generate_utils.py

@@ -113,6 +113,7 @@ def call_build_config(package_name: str, folder_name: str):
         os.environ.get("GH_TOKEN", None),
         packages=[package_name],
         build_conf=True,
+        template_names=["README.md", "__init__.py"],
     )
     # Replace this check_call by in process equivalent call, for better debugging
     # check_call(
@@ -156,10 +157,19 @@ def generate_packaging_files(package_name, folder_name):
             sdk_packaging_toml.rename(pyproject_toml)
 
     if "azure-mgmt-" in package_name:
+        # if codegen generate pyproject.toml instead of setup.py, we delete existing setup.py
+        setup_py = output_path / "setup.py"
+        if setup_py.exists():
+            _LOGGER.info(f"delete {setup_py} since codegen generate pyproject.toml")
+            with open(pyproject_toml, "rb") as f:
+                pyproject_content = toml.load(f)
+            if pyproject_content.get("project"):
+                setup_py.unlink()
+
         call_build_config(package_name, folder_name)
     else:
-        if not (output_path / CONF_NAME).exists():
-            with open(output_path / CONF_NAME, "w") as file_out:
+        if not pyproject_toml.exists():
+            with open(pyproject_toml, "w") as file_out:
                 file_out.write("[packaging]\nauto_update = false")
 
     # add ci.yaml
@@ -243,7 +253,38 @@ def judge_tag_preview(path: str, package_name: str) -> bool:
             _LOGGER.info(f"can not open {file}")
             continue
 
+        skip_decorator_block = False
+        decorator_depth = 0
+
         for line in list_in:
+            # skip the code of decorator @api_version_validation
+            stripped_line = line.strip()
+
+            # Check if we're starting an @api_version_validation decorator block
+            if "@api_version_validation" in stripped_line:
+                skip_decorator_block = True
+                decorator_depth = 0
+                continue
+
+            # If we're in a decorator block, track parentheses depth
+            if skip_decorator_block:
+                decorator_depth += stripped_line.count("(") - stripped_line.count(")")
+                # If we've closed all parentheses and hit a function definition, skip until next function/class
+                if decorator_depth == 0 and (
+                    stripped_line.startswith("def ") or stripped_line.startswith("async def ")
+                ):
+                    continue
+                # If we hit another decorator or function/class definition after closing parentheses, stop skipping
+                if decorator_depth == 0 and (
+                    stripped_line.startswith("@")
+                    or stripped_line.startswith("def ")
+                    or stripped_line.startswith("async def ")
+                    or stripped_line.startswith("class ")
+                ):
+                    skip_decorator_block = False
+                else:
+                    continue
+
             if "DEFAULT_API_VERSION = " in line:
                 default_api_version += line.split("=")[-1].strip("\n")  # collect all default api version
             if default_api_version == "" and "api_version" in line and "method_added_on" not in line:
@@ -462,13 +503,11 @@ def gen_typespec(
             tsp_dir = (Path(spec_folder) / typespec_relative_path).resolve()
             repo_url = rest_repo_url.replace("https://github.com/", "")
             tspconfig = tsp_dir / "tspconfig.yaml"
-            if tspconfig.exists():
+            if tspconfig.exists() and api_version:
                 with open(tspconfig, "r") as file_in:
                     content = yaml.safe_load(file_in)
                     if content.get("options", {}).get("@azure-tools/typespec-python"):
-                        content["options"]["@azure-tools/typespec-python"]["keep-setup-py"] = True
-                        if api_version:
-                            content["options"]["@azure-tools/typespec-python"]["api-version"] = api_version
+                        content["options"]["@azure-tools/typespec-python"]["api-version"] = api_version
                 with open(tspconfig, "w") as file_out:
                     yaml.dump(content, file_out)
             cmd = f"tsp-client init --tsp-config {tsp_dir} --local-spec-repo {tsp_dir} --commit {head_sha} --repo {repo_url}"

eng/tools/azure-sdk-tools/packaging_tools/package_utils.py

@@ -19,19 +19,13 @@
 from . import build_packaging
 from .change_log import main as change_log_main
 
-DEFAULT_DEST_FOLDER = "./dist"
-
 _LOGGER = logging.getLogger(__name__)
 
 
 # prefolder: "sdk/compute"; name: "azure-mgmt-compute"
-def create_package(prefolder, name, dest_folder=DEFAULT_DEST_FOLDER):
+def create_package(prefolder, name):
     absdirpath = Path(prefolder, name).absolute()
-    check_call(["python", "setup.py", "bdist_wheel", "-d", dest_folder], cwd=absdirpath)
-    check_call(
-        ["python", "setup.py", "sdist", "--format", "zip", "-d", dest_folder],
-        cwd=absdirpath,
-    )
+    check_call(["python", "-m", "build"], cwd=absdirpath)
 
 
 @return_origin_path
@@ -233,23 +227,47 @@ def check_file_with_packaging_tool(self):
 
         # add `title` and update `is_stable` in pyproject.toml
         pyproject_toml = Path(self.whole_package_name) / "pyproject.toml"
+        is_stable = self.tag_is_stable and self.next_version != "1.0.0b1"
         if pyproject_toml.exists():
             with open(pyproject_toml, "rb") as fd:
                 toml_data = toml.load(fd)
             if "packaging" not in toml_data:
                 toml_data["packaging"] = {}
             if title and not toml_data["packaging"].get("title"):
                 toml_data["packaging"]["title"] = title
-            toml_data["packaging"]["is_stable"] = self.tag_is_stable and self.next_version != "1.0.0b1"
+            toml_data["packaging"]["is_stable"] = is_stable
             with open(pyproject_toml, "wb") as fd:
                 tomlw.dump(toml_data, fd)
             _LOGGER.info(f"Update {pyproject_toml} successfully")
         else:
             _LOGGER.info(f"{os.getcwd()}/{pyproject_toml} does not exist")
 
-        build_packaging(output_folder=".", packages=[self.whole_package_name], build_conf=True)
+        build_packaging(
+            output_folder=".",
+            packages=[self.whole_package_name],
+            build_conf=True,
+            template_names=["README.md", "__init__.py"],
+        )
         _LOGGER.info("packaging_tools --build-conf successfully")
 
+        if pyproject_toml.exists():
+            stable_classifier = "Development Status :: 5 - Production/Stable"
+            beta_classifier = "Development Status :: 4 - Beta"
+
+            def edit_file(content: List[str]):
+                for i in range(0, len(content)):
+                    if "Development Status" in content[i]:
+                        if is_stable and beta_classifier in content[i]:
+                            content[i] = content[i].replace(beta_classifier, stable_classifier)
+                            _LOGGER.info(f"Replace '{beta_classifier}' with '{stable_classifier}' in {pyproject_toml}")
+                        if (not is_stable) and stable_classifier in content[i]:
+                            content[i] = content[i].replace(stable_classifier, beta_classifier)
+                            _LOGGER.info(f"Replace '{stable_classifier}' with '{beta_classifier}' in {pyproject_toml}")
+                        break
+
+            modify_file(str(pyproject_toml), edit_file)
+            _LOGGER.info(f"Check {pyproject_toml} for classifiers successfully")
+
     def sdk_code_path(self) -> str:
         return str(Path(f"sdk/{self.sdk_folder}/{self.whole_package_name}"))
 

eng/tools/azure-sdk-tools/packaging_tools/templates/packaging_files/__init__.py

@@ -1 +1 @@
-__path__ = __import__("pkgutil").extend_path(__path__, __name__)
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)  # type: ignore

scripts/auto_release/requirement.txt

@@ -8,6 +8,6 @@ fastcore==1.3.25
 tox==4.15.0
 wheel==0.43.0
 setuptools==78.1.0
-build==1.0.3
+build==1.3.0
 tomli-w==1.0.0
 tomli==2.2.1

scripts/automation_init.sh

@@ -7,6 +7,7 @@ pip install tox==4.15.0 > /dev/null
 pip install wheel==0.43.0 > /dev/null
 pip install setuptools==78.1.0 > /dev/null
 pip install setuptools-scm==8.3.0 > /dev/null
+pip install build==1.3.0 > /dev/null
 
 # install tsp-client globally (local install may interfere with tooling)
 echo Install tsp-client

swagger_to_sdk_config_autorest.json

@@ -9,7 +9,7 @@
       "include-x-ms-examples-original-file": true,
       "generate-sample": true,
       "generate-test": true,
-      "keep-setup-py": true
+      "package-mode": "azure-mgmt"
     },
     "advanced_options": {
       "create_sdk_pull_requests": true,