From f579c34185bb4c8ea97ac957cce00ca082f03008 Mon Sep 17 00:00:00 2001 From: jennypng <63012604+JennyPng@users.noreply.github.com> Date: Thu, 20 Nov 2025 12:46:06 -0800 Subject: [PATCH 1/3] minor bug fix --- eng/tools/azure-sdk-tools/azpysdk/sphinx.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/tools/azure-sdk-tools/azpysdk/sphinx.py b/eng/tools/azure-sdk-tools/azpysdk/sphinx.py index e89a3460fdac..8ee6f3757df1 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/sphinx.py +++ b/eng/tools/azure-sdk-tools/azpysdk/sphinx.py @@ -282,7 +282,7 @@ def sphinx_build( try: logger.info("Sphinx build command: {}".format(command_array)) - self.run_venv_command(executable, command_array, cwd=package_dir, check=True, append_executable=False) + self.run_venv_command(executable, command_array, cwd=package_dir, check=True, append_executable=False, immediately_dump=True) except CalledProcessError as e: logger.error("sphinx-build failed for path {} exited with error {}".format(target_dir, e.returncode)) if in_analyze_weekly(): @@ -306,14 +306,14 @@ def mgmt_apidoc(self, output_dir: str, target_folder: str, executable: str) -> i try: logger.info("Command to generate management sphinx sources: {}".format(command_array)) - self.run_venv_command(executable, command_array, cwd=target_folder, check=True, append_executable=False) + self.run_venv_command(executable, command_array, cwd=target_folder, check=True, append_executable=False, immediately_dump=True) except CalledProcessError as e: logger.error("script failed for path {} exited with error {}".format(output_dir, e.returncode)) return 1 return 0 def sphinx_apidoc(self, output_dir: str, target_dir: str, namespace: str, executable: str) -> int: - working_doc_folder = os.path.join(output_dir, "doc") + working_doc_folder = os.path.join(target_dir, "doc") command_array = [ "sphinx-apidoc", "--no-toc", From 6d9ca337853be6b0dfb51d8fa708656f8808b0d7 Mon Sep 17 00:00:00 2001 From: jennypng <63012604+JennyPng@users.noreply.github.com> Date: Fri, 21 Nov 2025 16:21:46 -0800 Subject: [PATCH 2/3] unzip --- eng/tools/azure-sdk-tools/azpysdk/sphinx.py | 77 ++++++++++++++++----- 1 file changed, 59 insertions(+), 18 deletions(-) diff --git a/eng/tools/azure-sdk-tools/azpysdk/sphinx.py b/eng/tools/azure-sdk-tools/azpysdk/sphinx.py index 8ee6f3757df1..acff7b39c6e5 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/sphinx.py +++ b/eng/tools/azure-sdk-tools/azpysdk/sphinx.py @@ -3,6 +3,8 @@ import sys import shutil import glob +import zipfile +import tarfile from typing import Optional, List from subprocess import CalledProcessError, check_call @@ -48,6 +50,39 @@ generate_mgmt_script = os.path.join(REPO_ROOT, "doc/sphinx/generate_doc.py") +def unzip_sdist_to_directory(containing_folder: str) -> str: + zips = glob.glob(os.path.join(containing_folder, "*.zip")) + + if zips: + return unzip_file_to_directory(zips[0], containing_folder) + else: + tars = glob.glob(os.path.join(containing_folder, "*.tar.gz")) + return unzip_file_to_directory(tars[0], containing_folder) + + +def unzip_file_to_directory(path_to_zip_file: str, extract_location: str) -> str: + if path_to_zip_file.endswith(".zip"): + with zipfile.ZipFile(path_to_zip_file, "r") as zip_ref: + zip_ref.extractall(extract_location) + extracted_dir = os.path.basename(os.path.splitext(path_to_zip_file)[0]) + return os.path.join(extract_location, extracted_dir) + else: + with tarfile.open(path_to_zip_file) as tar_ref: + tar_ref.extractall(extract_location) + extracted_dir = os.path.basename(path_to_zip_file).replace(".tar.gz", "") + return os.path.join(extract_location, extracted_dir) + + +def move_and_rename(source_location: str) -> str: + new_location = os.path.join(os.path.dirname(source_location), "unzipped") + + if os.path.exists(new_location): + shutil.rmtree(new_location) + + os.rename(source_location, new_location) + return new_location + + # env prep helper functions def create_index_file(readme_location: str, package_rst: str) -> str: readme_ext = os.path.splitext(readme_location)[1] @@ -220,22 +255,25 @@ def run(self, args: argparse.Namespace) -> int: logger.info(f"Running sphinx against {package_name}") # prep env for sphinx - doc_folder = os.path.join(staging_directory, "docgen") site_folder = os.path.join(package_dir, "website") + doc_folder = os.path.join(staging_directory, "docgen") if should_build_docs(package_name): - create_index(doc_folder, package_dir, parsed.namespace) + source_location = move_and_rename(unzip_sdist_to_directory(staging_directory)) + doc_folder = os.path.join(source_location, "docgen") + create_index(doc_folder, source_location, parsed.namespace) write_version(site_folder, parsed.version) else: logger.info("Skipping sphinx prep for {}".format(package_name)) # run apidoc + output_dir = os.path.join(staging_directory, "unzipped/docgen") if should_build_docs(parsed.name): if is_mgmt_package(parsed.name): - results.append(self.mgmt_apidoc(doc_folder, package_dir, executable)) + results.append(self.mgmt_apidoc(output_dir, package_dir, executable)) else: - results.append(self.sphinx_apidoc(staging_directory, package_dir, parsed.namespace, executable)) + results.append(self.sphinx_apidoc(staging_directory, parsed.namespace, executable)) else: logger.info("Skipping sphinx source generation for {}".format(parsed.name)) @@ -282,7 +320,9 @@ def sphinx_build( try: logger.info("Sphinx build command: {}".format(command_array)) - self.run_venv_command(executable, command_array, cwd=package_dir, check=True, append_executable=False, immediately_dump=True) + self.run_venv_command( + executable, command_array, cwd=package_dir, check=True, append_executable=False, immediately_dump=True + ) except CalledProcessError as e: logger.error("sphinx-build failed for path {} exited with error {}".format(target_dir, e.returncode)) if in_analyze_weekly(): @@ -306,33 +346,34 @@ def mgmt_apidoc(self, output_dir: str, target_folder: str, executable: str) -> i try: logger.info("Command to generate management sphinx sources: {}".format(command_array)) - self.run_venv_command(executable, command_array, cwd=target_folder, check=True, append_executable=False, immediately_dump=True) + self.run_venv_command( + executable, command_array, cwd=target_folder, check=True, append_executable=False, immediately_dump=True + ) except CalledProcessError as e: logger.error("script failed for path {} exited with error {}".format(output_dir, e.returncode)) return 1 return 0 - def sphinx_apidoc(self, output_dir: str, target_dir: str, namespace: str, executable: str) -> int: - working_doc_folder = os.path.join(target_dir, "doc") + def sphinx_apidoc(self, target_dir: str, namespace: str, executable: str) -> int: + working_doc_folder = os.path.join(target_dir, "unzipped/doc") command_array = [ "sphinx-apidoc", "--no-toc", "--module-first", "-o", - os.path.join(output_dir, "docgen"), # This is the output folder - os.path.join(target_dir, ""), # This is the input folder - os.path.join(target_dir, "test*"), # This argument and below are "exclude" directory arguments - os.path.join(target_dir, "example*"), - os.path.join(target_dir, "sample*"), - os.path.join(target_dir, "setup.py"), - os.path.join(target_dir, "conftest.py"), + os.path.join(target_dir, "unzipped/docgen"), # This is the output folder + os.path.join(target_dir, "unzipped/"), # This is the input folder + os.path.join(target_dir, "unzipped/test*"), # This argument and below are "exclude" directory arguments + os.path.join(target_dir, "unzipped/example*"), + os.path.join(target_dir, "unzipped/sample*"), + os.path.join(target_dir, "unzipped/setup.py"), ] try: # if a `doc` folder exists, just leverage the sphinx sources found therein. if os.path.exists(working_doc_folder): logger.info("Copying files into sphinx source folder.") - copy_existing_docs(working_doc_folder, os.path.join(output_dir, "docgen")) + copy_existing_docs(working_doc_folder, os.path.join(target_dir, "unzipped/docgen")) # otherwise, we will run sphinx-apidoc to generate the sources else: @@ -340,7 +381,7 @@ def sphinx_apidoc(self, output_dir: str, target_dir: str, namespace: str, execut self.run_venv_command(executable, command_array, cwd=target_dir, check=True, append_executable=False) # We need to clean "azure.rst", and other RST before the main namespaces, as they are never # used and will log as a warning later by sphinx-build, which is blocking strict_sphinx - base_path = Path(os.path.join(output_dir, "docgen/")) + base_path = Path(os.path.join(target_dir, "unzipped/docgen/")) namespace = namespace.rpartition(".")[0] while namespace: rst_file_to_delete = base_path / f"{namespace}.rst" @@ -348,6 +389,6 @@ def sphinx_apidoc(self, output_dir: str, target_dir: str, namespace: str, execut rst_file_to_delete.unlink(missing_ok=True) namespace = namespace.rpartition(".")[0] except CalledProcessError as e: - logger.error("sphinx-apidoc failed for path {} exited with error {}".format(output_dir, e.returncode)) + logger.error("sphinx-apidoc failed for path {} exited with error {}".format(target_dir, e.returncode)) return 1 return 0 From ab0b467d00ecac33399955048b443403a92d1607 Mon Sep 17 00:00:00 2001 From: jennypng <63012604+JennyPng@users.noreply.github.com> Date: Fri, 21 Nov 2025 17:00:35 -0800 Subject: [PATCH 3/3] cut --- eng/pipelines/templates/steps/build-extended-artifacts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/steps/build-extended-artifacts.yml b/eng/pipelines/templates/steps/build-extended-artifacts.yml index 05d923b93030..7bff160aefe7 100644 --- a/eng/pipelines/templates/steps/build-extended-artifacts.yml +++ b/eng/pipelines/templates/steps/build-extended-artifacts.yml @@ -67,11 +67,11 @@ steps: displayName: 'Generate Docs' condition: and(succeededOrFailed(), ${{parameters.BuildDocs}}) inputs: - scriptPath: 'scripts/devops_tasks/dispatch_tox.py' + scriptPath: 'eng/scripts/dispatch_checks.py' arguments: >- "$(TargetingString)" --service="${{ parameters.ServiceDirectory }}" - --toxenv=sphinx + --checks="sphinx" --wheel_dir="$(Build.ArtifactStagingDirectory)" - ${{ if eq(parameters.RunApiStubGen, 'true') }}: