diff --git a/.generator/Dockerfile b/.generator/Dockerfile index b1f49582c2c3..3e0470e642bf 100644 --- a/.generator/Dockerfile +++ b/.generator/Dockerfile @@ -16,22 +16,15 @@ # This stage installs all build dependencies and compiles all Python versions. FROM marketplace.gcr.io/google/ubuntu2404 AS builder -# TODO(https://github.com/googleapis/librarian/issues/901): Install the necssary dependencies and build tools. RUN apt-get update && \ apt-get install -y --no-install-recommends \ # Essential for compiling C code build-essential \ - # For downloading secure files + # For downloading and extracting secure files git \ wget \ ca-certificates \ - # For running bazelisk commands - openjdk-17-jdk \ - zip \ - unzip \ - # To avoid bazel error - # "python interpreter `python3` not found in PATH" - python3-dev \ + unzip \ # --- Critical libraries for a complete Python build --- libssl-dev \ zlib1g-dev \ @@ -43,9 +36,6 @@ RUN apt-get update && \ && apt-get clean && \ rm -rf /var/lib/apt/lists/* -# Set up environment variables for tool versions to make updates easier. -ENV BAZELISK_VERSION=v1.26.0 - # Install multiple Python versions from source. `make altinstall` is used to # prevent replacing the system's default python binary. # TODO(http://github.com/googleapis/gapic-generator-python/issues/2435): Remove `3.10.18` when the linked issue is resolved. @@ -68,77 +58,48 @@ RUN wget --no-check-certificate -O /tmp/get-pip.py 'https://bootstrap.pypa.io/ge done && \ rm /tmp/get-pip.py -# Install Bazelisk -RUN wget https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-linux-amd64 -O /usr/local/bin/bazelisk && \ - chmod +x /usr/local/bin/bazelisk - -# Set the working directory for build-related tasks. -WORKDIR /app - -# Create the group and user, but only if they don't already exist. -ARG UID=1000 -ARG GID=1000 - -RUN if ! getent group $GID > /dev/null; then \ - groupadd -g $GID myuser; \ - fi && \ - if ! getent passwd $UID > /dev/null; then \ - useradd -u $UID -g $GID -ms /bin/bash myuser; \ - fi - -# Set ownership of the app directory now, before we copy files into it. -RUN mkdir -p /app && chown $UID:$GID /app - -# We'll point both to the /bazel_cache directory which will be mounted as a volume. -ENV BAZELISK_HOME="/bazel_cache/bazelisk" -ENV BAZEL_HOME="/bazel_cache/bazel" - -# Ensure the cache directories within the non-root user's context exist and are writable. -# This is crucial as Bazel creates subdirectories under BAZEL_HOME. -RUN mkdir -p ${BAZEL_HOME}/_bazel_ubuntu/cache/repos \ - ${BAZEL_HOME}/_bazel_ubuntu/output_base \ - ${BAZELISK_HOME} && \ - chown -R $UID:$GID ${BAZEL_HOME} ${BAZELISK_HOME} - RUN /usr/local/bin/python3.9 -m venv bazel_env RUN . bazel_env/bin/activate -RUN git clone https://github.com/googleapis/googleapis.git \ - && cd googleapis \ - && bazelisk --output_base=/bazel_cache/_bazel_ubuntu/output_base build --disk_cache=/bazel_cache/_bazel_ubuntu/cache/repos --incompatible_strict_action_env //google/cloud/language/v1:language-v1-py +# Download/extract protoc +RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v25.3/protoc-25.3-linux-x86_64.zip +RUN unzip protoc-25.3-linux-x86_64.zip -d protoc -# TODO(https://github.com/googleapis/librarian/issues/904): Install protoc for gencode. +# Download/extract pandoc +# Pandoc is required by gapic-generator-python for parsing documentation +RUN wget https://github.com/jgm/pandoc/releases/download/3.7.0.2/pandoc-3.7.0.2-linux-amd64.tar.gz +RUN tar -xvf pandoc-3.7.0.2-linux-amd64.tar.gz + +# Download synthtool +RUN git clone --depth 1 https://github.com/googleapis/synthtool.git synthtool # --- Final Stage --- # This stage creates the lightweight final image, copying only the # necessary artifacts from the builder stage. FROM marketplace.gcr.io/google/ubuntu2404 +# Tell synthtool to pull templates from this docker image instead of from +# the live repo. +ENV SYNTHTOOL_TEMPLATES="/synthtool/synthtool/gcp/templates" + # Install only the essential runtime libraries for Python. # These are the non "-dev" versions of the libraries used in the builder. RUN apt-get update && \ apt-get install -y --no-install-recommends \ - ca-certificates \ - git \ - libssl3 \ - zlib1g \ - libbz2-1.0 \ - libffi8 \ + # This is needed to avoid the following error: + # `ImportError: libsqlite3.so.0: cannot open shared object file: No such file or directory`. + # `libsqlite3-0` is used by the `coverage` PyPI package which is used when testing libraries libsqlite3-0 \ - libreadline8 \ - # For running bazelisk commands - openjdk-17-jdk \ - # To avoid bazel error - # "python interpreter `python3` not found in PATH" - python3-dev \ - # To avoid bazel error - # "Cannot find gcc or CC; either correct your path or set the CC environment variable" - build-essential \ - # To avoid bazel error - # unzip command not found - unzip \ - && apt-get clean && \ - rm -rf /var/lib/apt/lists/* + && apt-get clean autoclean \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* \ + && rm -f /var/cache/apt/archives/*.deb + +COPY --from=builder protoc/bin /usr/local/bin +COPY --from=builder protoc/include /usr/local/include + +COPY --from=builder pandoc-3.7.0.2/bin /usr/local/bin +COPY --from=builder synthtool /synthtool # Copy all Python interpreters, their pip executables, and their standard libraries from the builder. COPY --from=builder /usr/local/bin/python3.9 /usr/local/bin/ @@ -151,26 +112,16 @@ COPY --from=builder /usr/local/lib/python3.10 /usr/local/lib/python3.10 COPY --from=builder /usr/local/bin/python3.13 /usr/local/bin/ COPY --from=builder /usr/local/lib/python3.13 /usr/local/lib/python3.13 -# Copy the bazelisk executable from the builder. -COPY --from=builder /usr/local/bin/bazelisk /usr/local/bin/ - -# Copy bazel cache from the builder. -COPY --from=builder /bazel_cache /bazel_cache -RUN chmod -R 777 /bazel_cache - # Set the working directory in the container. WORKDIR /app -# Create a virtual env and set the Path to fix the missing nox error -# when running the post processor changes. -RUN /usr/local/bin/python3.9 -m venv bazel_env -RUN . bazel_env/bin/activate - -ENV PATH=/app/bazel_env/bin:$PATH - -RUN git clone --depth 1 https://github.com/googleapis/synthtool.git /tmp/synthtool && \ -bazel_env/bin/python3.9 -m pip install /tmp/synthtool nox && \ -rm -rf /tmp/synthtool +# Install dependencies of the CLI such as click. +# Install gapic-generator which is used to generate libraries. +# Install nox which is used for running client library tests. +# Install starlark-pyo3 which is used to parse BUILD.bazel files. +COPY .generator/requirements.in . +RUN python3.9 -m pip install -r requirements.in +RUN python3.9 -m pip install /synthtool # Install build which is used to get the metadata of package config files. COPY .generator/requirements.in . @@ -180,4 +131,8 @@ RUN python3.9 -m pip install -r requirements.in COPY .generator/cli.py . RUN chmod a+rx ./cli.py +# Copy the script used to parse BUILD.bazel files. +COPY .generator/parse_googleapis_content.py . +RUN chmod a+rx ./parse_googleapis_content.py + ENTRYPOINT ["python3.9", "./cli.py"] diff --git a/.generator/cli.py b/.generator/cli.py index 17ba6602465c..6c422bcf6853 100644 --- a/.generator/cli.py +++ b/.generator/cli.py @@ -22,12 +22,14 @@ import shutil import subprocess import sys +import tempfile import yaml from datetime import datetime from pathlib import Path from typing import Dict, List - import build.util +import parse_googleapis_content + try: import synthtool @@ -118,46 +120,6 @@ def handle_configure(): logger.info("'configure' command executed.") -def _determine_bazel_rule(api_path: str, source: str) -> str: - """Finds a Bazel rule by parsing the BUILD.bazel file directly. - - Args: - api_path (str): The API path, e.g., 'google/cloud/language/v1'. - source(str): The path to the root of the Bazel workspace. - - Returns: - str: The discovered Bazel rule, e.g., '//google/cloud/language/v1:language-v1-py'. - - Raises: - ValueError: If the file can't be processed or no matching rule is found. - """ - logger.info(f"Determining Bazel rule for api_path: '{api_path}' by parsing file.") - try: - build_file_path = os.path.join(source, api_path, "BUILD.bazel") - - with open(build_file_path, "r") as f: - content = f.read() - - match = re.search(r'name\s*=\s*"([^"]+-py)"', content) - - # This check is for a logical failure (no match), not a runtime exception. - # It's good to keep it for clear error messaging. - if not match: # pragma: NO COVER - raise ValueError( - f"No Bazel rule with a name ending in '-py' found in {build_file_path}" - ) - - rule_name = match.group(1) - bazel_rule = f"//{api_path}:{rule_name}" - - logger.info(f"Found Bazel rule: {bazel_rule}") - return bazel_rule - except Exception as e: - raise ValueError( - f"Failed to determine Bazel rule for '{api_path}' by parsing." - ) from e - - def _get_library_id(request_data: Dict) -> str: """Retrieve the library id from the given request dictionary @@ -176,107 +138,6 @@ def _get_library_id(request_data: Dict) -> str: return library_id -def _build_bazel_target(bazel_rule: str, source: str): - """Executes `bazelisk build` on a given Bazel rule. - - Args: - bazel_rule(str): The Bazel rule to build. - source(str): The path to the root of the Bazel workspace. - - Raises: - ValueError: If the subprocess call fails. - """ - logger.info(f"Executing build for rule: {bazel_rule}") - try: - # We're using the prewarmed bazel cache from the docker image to speed up the bazelisk commands. - # Previously built artifacts are stored in `/bazel_cache/_bazel_ubuntu/output_base` and will be - # used to speed up the build. `disk_cache` is used as the 'remote cache' and is also prewarmed as part of - # the docker image. - # See https://bazel.build/remote/caching#disk-cache which explains using a file system as a 'remote cache'. - command = [ - "bazelisk", - "--output_base=/bazel_cache/_bazel_ubuntu/output_base", - "build", - "--disk_cache=/bazel_cache/_bazel_ubuntu/cache/repos", - "--incompatible_strict_action_env", - bazel_rule, - ] - subprocess.run( - command, - cwd=source, - text=True, - check=True, - ) - logger.info(f"Bazel build for {bazel_rule} rule completed successfully.") - except Exception as e: - raise ValueError(f"Bazel build for {bazel_rule} rule failed.") from e - - -def _locate_and_extract_artifact( - bazel_rule: str, - library_id: str, - source: str, - output: str, - api_path: str, -): - """Finds and extracts the tarball artifact from a Bazel build. - - Args: - bazel_rule(str): The Bazel rule that was built. - library_id(str): The ID of the library being generated. - source(str): The path to the root of the Bazel workspace. - output(str): The path to the location where generated output - should be stored. - api_path(str): The API path for the artifact - - Raises: - ValueError: If failed to locate or extract artifact. - """ - try: - # 1. Find the bazel-bin output directory. - logger.info("Locating Bazel output directory...") - # Previously built artifacts are stored in `/bazel_cache/_bazel_ubuntu/output_base`. - # See `--output_base` in `_build_bazel_target` - info_command = [ - "bazelisk", - "--output_base=/bazel_cache/_bazel_ubuntu/output_base", - "info", - "bazel-bin", - ] - result = subprocess.run( - info_command, - cwd=source, - text=True, - check=True, - capture_output=True, - ) - bazel_bin_path = result.stdout.strip() - - # 2. Construct the path to the generated tarball. - rule_path, rule_name = bazel_rule.split(":") - tarball_name = f"{rule_name}.tar.gz" - tarball_path = os.path.join(bazel_bin_path, rule_path.strip("/"), tarball_name) - logger.info(f"Found artifact at: {tarball_path}") - - # 3. Create a staging directory. - api_version = api_path.split("/")[-1] - staging_dir = os.path.join(output, "owl-bot-staging", library_id, api_version) - os.makedirs(staging_dir, exist_ok=True) - logger.info(f"Preparing staging directory: {staging_dir}") - - # 4. Extract the artifact. - extract_command = ["tar", "-xvf", tarball_path, "--strip-components=1"] - subprocess.run( - extract_command, cwd=staging_dir, capture_output=True, text=True, check=True - ) - logger.info(f"Artifact {tarball_path} extracted successfully.") - - except Exception as e: - raise ValueError( - f"Failed to locate or extract artifact for {bazel_rule} rule" - ) from e - - def _run_post_processor(output: str, library_id: str): """Runs the synthtool post-processor on the output directory. @@ -401,25 +262,150 @@ def handle_generate( # Read a generate-request.json file request_data = _read_json_file(f"{librarian}/{GENERATE_REQUEST_FILE}") library_id = _get_library_id(request_data) - for api in request_data.get("apis", []): + apis_to_generate = request_data.get("apis", []) + for api in apis_to_generate: api_path = api.get("path") if api_path: - bazel_rule = _determine_bazel_rule(api_path, source) - _build_bazel_target(bazel_rule, source) - _locate_and_extract_artifact( - bazel_rule, library_id, source, output, api_path - ) - + _generate_api(api_path, library_id, source, output) _copy_files_needed_for_post_processing(output, input, library_id) _run_post_processor(output, library_id) _clean_up_files_after_post_processing(output, library_id) - except Exception as e: raise ValueError("Generation failed.") from e - logger.info("'generate' command executed.") +def _read_bazel_build_py_rule(api_path: str, source: str) -> Dict: + """ + Reads and parses the BUILD.bazel file to find the Python GAPIC rule content. + + Args: + api_path (str): The relative path to the API directory (e.g., 'google/cloud/language/v1'). + source (str): Path to the directory containing API protos. + + Returns: + Dict: A dictionary containing the parsed attributes of the `_py_gapic` rule. + """ + build_file_path = f"{source}/{api_path}/BUILD.bazel" + content = _read_text_file(build_file_path) + + result = parse_googleapis_content.parse_content(content) + py_gapic_entries = [key for key in result.keys() if key.endswith("_py_gapic")] + + # Assuming only one _py_gapic rule per BUILD file for a given language + return result[py_gapic_entries[0]] + + +def _get_api_generator_options(api_path: str, py_gapic_config: Dict) -> List[str]: + """ + Extracts generator options from the parsed Python GAPIC rule configuration. + + Args: + api_path (str): The relative path to the API directory. + py_gapic_config (Dict): The parsed attributes of the Python GAPIC rule. + + Returns: + List[str]: A list of formatted generator options (e.g., ['retry-config=...', 'transport=...']). + """ + generator_options = [] + + # Mapping of Bazel rule attributes to protoc-gen-python_gapic options + config_key_map = { + "grpc_service_config": "retry-config", + "rest_numeric_enums": "rest-numeric-enums", + "service_yaml": "service-yaml", + "transport": "transport", + } + + for bazel_key, protoc_key in config_key_map.items(): + config_value = py_gapic_config.get(bazel_key) + if config_value is not None: + if bazel_key in ("service_yaml", "grpc_service_config"): + # These paths are relative to the source root + generator_options.append(f"{protoc_key}={api_path}/{config_value}") + else: + # Other options use the value directly + generator_options.append(f"{protoc_key}={config_value}") + + # Add optional arguments + optional_arguments = py_gapic_config.get("opt_args", None) + if optional_arguments: + # opt_args in Bazel rule is already a list of strings + generator_options.extend(optional_arguments) + + return generator_options + + +def _determine_generator_command( + api_path: str, tmp_dir: str, generator_options: List[str] +) -> str: + """ + Constructs the full protoc command string. + + Args: + api_path (str): The relative path to the API directory. + tmp_dir (str): The temporary directory for protoc output. + generator_options (List[str]): Extracted generator options. + + Returns: + str: The complete protoc command string suitable for shell execution. + """ + # Start with the protoc base command. The glob pattern requires shell=True. + command_parts = [ + f"protoc {api_path}/*.proto", + f"--python_gapic_out={tmp_dir}", + ] + + if generator_options: + # Protoc options are passed as a comma-separated list to --python_gapic_opt. + option_string = "metadata," + ",".join(generator_options) + command_parts.append(f"--python_gapic_opt={option_string}") + + return " ".join(command_parts) + + +def _run_generator_command(generator_command: str, source: str): + """ + Executes the protoc generation command using subprocess. + + Args: + generator_command (str): The complete protoc command string. + source (str): Path to the directory where the command should be run (API protos root). + """ + # shell=True is required because the command string contains a glob pattern (*.proto) + subprocess.run( + [generator_command], + cwd=source, + shell=True, + check=True, + capture_output=True, + text=True, + ) + + +def _generate_api(api_path: str, library_id: str, source: str, output: str): + """ + Handles the generation and staging process for a single API path. + + Args: + api_path (str): The relative path to the API directory (e.g., 'google/cloud/language/v1'). + library_id (str): The ID of the library being generated. + source (str): Path to the directory containing API protos. + output (str): Path to the output directory where code should be staged. + """ + py_gapic_config = _read_bazel_build_py_rule(api_path, source) + generator_options = _get_api_generator_options(api_path, py_gapic_config) + + with tempfile.TemporaryDirectory() as tmp_dir: + generator_command = _determine_generator_command( + api_path, tmp_dir, generator_options + ) + _run_generator_command(generator_command, source) + api_version = api_path.split("/")[-1] + staging_dir = os.path.join(output, "owl-bot-staging", library_id, api_version) + shutil.copytree(tmp_dir, staging_dir) + + def _run_nox_sessions(library_id: str, repo: str): """Calls nox for all specified sessions. diff --git a/.generator/requirements.in b/.generator/requirements.in index 378eac25d311..d866c3e04009 100644 --- a/.generator/requirements.in +++ b/.generator/requirements.in @@ -1 +1,5 @@ +click +gapic-generator +nox +starlark-pyo3>=2025.1 build diff --git a/.generator/test_cli.py b/.generator/test_cli.py index dcac0df5a0c3..7e839f6bc610 100644 --- a/.generator/test_cli.py +++ b/.generator/test_cli.py @@ -29,25 +29,28 @@ GENERATE_REQUEST_FILE, BUILD_REQUEST_FILE, RELEASE_INIT_REQUEST_FILE, + SOURCE_DIR, STATE_YAML_FILE, LIBRARIAN_DIR, REPO_DIR, - _build_bazel_target, _clean_up_files_after_post_processing, _copy_files_needed_for_post_processing, _create_main_version_header, - _determine_bazel_rule, - _get_library_dist_name, + _determine_generator_command, _determine_library_namespace, + _generate_api, + _get_api_generator_options, + _get_library_dist_name, _get_library_id, _get_libraries_to_prepare_for_release, _get_previous_version, - _locate_and_extract_artifact, _process_changelog, _process_version_file, + _read_bazel_build_py_rule, _read_json_file, _read_text_file, _run_individual_session, + _run_generator_command, _run_nox_sessions, _run_post_processor, _update_changelog_for_library, @@ -95,6 +98,27 @@ }, ] +_MOCK_BAZEL_CONTENT = """load( + "@com_google_googleapis_imports//:imports.bzl", + "py_gapic_assembly_pkg", + "py_gapic_library", + "py_test", +) + +py_gapic_library( + name = "language_py_gapic", + srcs = [":language_proto"], + grpc_service_config = "language_grpc_service_config.json", + rest_numeric_enums = True, + service_yaml = "language_v1.yaml", + transport = "grpc+rest", + deps = [ + ], + opt_args = [ + "python-gapic-namespace=google.cloud", + ], +)""" + @pytest.fixture def mock_generate_request_file(tmp_path, monkeypatch): @@ -136,6 +160,18 @@ def mock_build_request_file(tmp_path, monkeypatch): return request_file +@pytest.fixture +def mock_build_bazel_file(tmp_path, monkeypatch): + """Creates the mock BUILD.bazel file at the correct path inside a temp dir.""" + bazel_build_path = f"{SOURCE_DIR}/google/cloud/language/v1/BUILD.bazel" + bazel_build_dir = tmp_path / Path(bazel_build_path).parent + os.makedirs(bazel_build_dir, exist_ok=True) + build_bazel_file = bazel_build_dir / os.path.basename(bazel_build_path) + + build_bazel_file.write_text(_MOCK_BAZEL_CONTENT) + return build_bazel_file + + @pytest.fixture def mock_generate_request_data_for_nox(): """Returns mock data for generate-request.json for nox tests.""" @@ -236,159 +272,171 @@ def test_handle_configure_success(caplog, mock_generate_request_file): assert "'configure' command executed." in caplog.text -def test_determine_bazel_rule_success(mocker, caplog): +def test_run_post_processor_success(mocker, caplog): """ - Tests the happy path of _determine_bazel_rule. + Tests that the post-processor helper calls the correct command. """ caplog.set_level(logging.INFO) - mock_content = 'name = "google-cloud-language-v1-py",\n' - mocker.patch("cli.open", mock_open(read_data=mock_content)) + mocker.patch("cli.SYNTHTOOL_INSTALLED", return_value=True) + mock_chdir = mocker.patch("cli.os.chdir") + mock_owlbot_main = mocker.patch( + "cli.synthtool.languages.python_mono_repo.owlbot_main" + ) + _run_post_processor("output", "google-cloud-language") - rule = _determine_bazel_rule("google/cloud/language/v1", "source") + mock_chdir.assert_called_once() - assert rule == "//google/cloud/language/v1:google-cloud-language-v1-py" - assert "Found Bazel rule" in caplog.text + mock_owlbot_main.assert_called_once_with("packages/google-cloud-language") + assert "Python post-processor ran successfully." in caplog.text -def test_build_bazel_target_success(mocker, caplog): - """ - Tests that the build helper logs success when the command runs correctly. - """ - caplog.set_level(logging.INFO) - mocker.patch("cli.subprocess.run", return_value=MagicMock(returncode=0)) - _build_bazel_target("mock/bazel:rule", "source") - assert "Bazel build for mock/bazel:rule rule completed successfully" in caplog.text +def test_read_bazel_build_py_rule_success(mocker, mock_build_bazel_file): + """Tests successful reading and parsing of a valid BUILD.bazel file.""" + api_path = "google/cloud/language/v1" + # Use the empty string as the source path, since the fixture has set the CWD to the temporary root. + source_dir = "source" + mocker.patch("cli._read_text_file", return_value=_MOCK_BAZEL_CONTENT) + # The fixture already creates the file, so we just need to call the function + py_gapic_config = _read_bazel_build_py_rule(api_path, source_dir) -def test_build_bazel_target_fails_to_find_rule_match(mocker, caplog): - """ - Tests that ValueError is raised if the subprocess command fails. - """ - caplog.set_level(logging.ERROR) - mock_content = '"google-cloud-language-v1-py",\n' - mocker.patch("cli.open", mock_open(read_data=mock_content)) + assert ( + "language_py_gapic" not in py_gapic_config + ) # Only rule attributes should be returned + assert py_gapic_config["grpc_service_config"] == "language_grpc_service_config.json" + assert py_gapic_config["rest_numeric_enums"] is True + assert py_gapic_config["transport"] == "grpc+rest" + assert py_gapic_config["opt_args"] == ["python-gapic-namespace=google.cloud"] + + +def test_get_api_generator_options_all_options(): + """Tests option extraction when all relevant fields are present.""" + api_path = "google/cloud/language/v1" + py_gapic_config = { + "grpc_service_config": "config.json", + "rest_numeric_enums": True, + "service_yaml": "service.yaml", + "transport": "grpc+rest", + "opt_args": ["single_arg", "another_arg"], + } + options = _get_api_generator_options(api_path, py_gapic_config) + + expected = [ + "retry-config=google/cloud/language/v1/config.json", + "rest-numeric-enums=True", + "service-yaml=google/cloud/language/v1/service.yaml", + "transport=grpc+rest", + "single_arg", + "another_arg", + ] + assert sorted(options) == sorted(expected) - with pytest.raises(ValueError): - _build_bazel_target("mock/bazel:rule", "source") +def test_get_api_generator_options_minimal_options(): + """Tests option extraction when only transport is present.""" + api_path = "google/cloud/minimal/v1" + py_gapic_config = { + "transport": "grpc", + } + options = _get_api_generator_options(api_path, py_gapic_config) -def test_build_bazel_target_fails_to_determine_rule(caplog): - """ - Tests that ValueError is raised if the subprocess command fails. - """ - caplog.set_level(logging.ERROR) - with pytest.raises(ValueError): - _build_bazel_target("mock/bazel:rule", "source") + expected = ["transport=grpc"] + assert options == expected -def test_build_bazel_target_fails(mocker, caplog): - """ - Tests that ValueError is raised if the subprocess command fails. - """ - caplog.set_level(logging.ERROR) - mock_content = '"google-cloud-language-v1-py",\n' - mocker.patch("cli.open", mock_open(read_data=mock_content)) +def test_determine_generator_command_with_options(): + """Tests command construction with options.""" + api_path = "google/cloud/test/v1" + tmp_dir = "/tmp/output/test" + options = ["transport=grpc", "custom_option=foo"] + command = _determine_generator_command(api_path, tmp_dir, options) - with pytest.raises(ValueError): - _build_bazel_target("mock/bazel:rule", "source") + expected_options = "--python_gapic_opt=metadata,transport=grpc,custom_option=foo" + expected_command = ( + f"protoc {api_path}/*.proto --python_gapic_out={tmp_dir} {expected_options}" + ) + assert command == expected_command -def test_determine_bazel_rule_command_fails(mocker, caplog): - """ - Tests that an exception is raised if the subprocess command fails. - """ - caplog.set_level(logging.INFO) - mocker.patch( - "cli.subprocess.run", - side_effect=subprocess.CalledProcessError(1, "cmd", stderr="Bazel error"), +def test_determine_generator_command_no_options(): + """Tests command construction without extra options.""" + api_path = "google/cloud/test/v1" + tmp_dir = "/tmp/output/test" + options = [] + command = _determine_generator_command(api_path, tmp_dir, options) + + # Note: 'metadata' is always included if options list is empty or not + # only if `generator_options` is not empty. If it is empty, the result is: + expected_command_no_options = ( + f"protoc {api_path}/*.proto --python_gapic_out={tmp_dir}" ) + assert command == expected_command_no_options - with pytest.raises(ValueError): - _determine_bazel_rule("google/cloud/language/v1", "source") - assert "Found Bazel rule" not in caplog.text +def test_run_generator_command_success(mocker): + """Tests successful execution of the protoc command.""" + mock_run = mocker.patch( + "cli.subprocess.run", return_value=MagicMock(stdout="ok", stderr="", check=True) + ) + command = "protoc api/*.proto --python_gapic_out=/tmp/out" + source = "/src" + _run_generator_command(command, source) -def test_locate_and_extract_artifact_success(mocker, caplog): - """ - Tests that the artifact helper calls the correct sequence of commands. - """ - caplog.set_level(logging.INFO) - mock_info_result = MagicMock(stdout="/path/to/bazel-bin\n") - mock_tar_result = MagicMock(returncode=0) - mocker.patch("cli.subprocess.run", side_effect=[mock_info_result, mock_tar_result]) - mock_makedirs = mocker.patch("cli.os.makedirs") - _locate_and_extract_artifact( - "//google/cloud/language/v1:rule-py", - "google-cloud-language", - "source", - "output", - "google/cloud/language/v1", - ) - assert ( - "Found artifact at: /path/to/bazel-bin/google/cloud/language/v1/rule-py.tar.gz" - in caplog.text + mock_run.assert_called_once_with( + [command], cwd=source, shell=True, check=True, capture_output=True, text=True ) - assert ( - "Preparing staging directory: output/owl-bot-staging/google-cloud-language" - in caplog.text - ) - assert ( - "Artifact /path/to/bazel-bin/google/cloud/language/v1/rule-py.tar.gz extracted successfully" - in caplog.text - ) - mock_makedirs.assert_called_once() -def test_locate_and_extract_artifact_fails(mocker, caplog): - """ - Tests that an exception is raised if the subprocess command fails. - """ - caplog.set_level(logging.INFO) - mocker.patch( +def test_run_generator_command_failure(mocker): + """Tests failure when protoc command returns a non-zero exit code.""" + mock_run = mocker.patch( "cli.subprocess.run", - side_effect=subprocess.CalledProcessError(1, "cmd", stderr="Bazel error"), + side_effect=subprocess.CalledProcessError(1, "protoc", stderr="error"), ) + command = "protoc api/*.proto --python_gapic_out=/tmp/out" + source = "/src" - with pytest.raises(ValueError): - _locate_and_extract_artifact( - "//google/cloud/language/v1:rule-py", - "google-cloud-language", - "source", - "output", - "google/cloud/language/v1", - ) + with pytest.raises(subprocess.CalledProcessError): + _run_generator_command(command, source) -def test_run_post_processor_success(mocker, caplog): - """ - Tests that the post-processor helper calls the correct command. - """ +def test_generate_api_success(mocker, caplog): caplog.set_level(logging.INFO) - mocker.patch("cli.SYNTHTOOL_INSTALLED", return_value=True) - mock_chdir = mocker.patch("cli.os.chdir") - mock_owlbot_main = mocker.patch( - "cli.synthtool.languages.python_mono_repo.owlbot_main" + + API_PATH = "google/cloud/language/v1" + LIBRARY_ID = "google-cloud-language" + SOURCE = "source" + OUTPUT = "output" + + mock_read_bazel_build_py_rule = mocker.patch("cli._read_bazel_build_py_rule") + mock_get_api_generator_options = mocker.patch("cli._get_api_generator_options") + mock_determine_generator_command = mocker.patch( + "cli._determine_generator_command" ) - _run_post_processor("output", "google-cloud-language") + mock_run_generator_command = mocker.patch( + "cli._run_generator_command" + ) + mock_shutil_copytree = mocker.patch("shutil.copytree") - mock_chdir.assert_called_once() + _generate_api(API_PATH, LIBRARY_ID, SOURCE, OUTPUT) - mock_owlbot_main.assert_called_once_with("packages/google-cloud-language") - assert "Python post-processor ran successfully." in caplog.text + mock_read_bazel_build_py_rule.assert_called_once() + mock_get_api_generator_options.assert_called_once() + mock_determine_generator_command.assert_called_once() + mock_run_generator_command.assert_called_once() + mock_shutil_copytree.assert_called_once() -def test_handle_generate_success(caplog, mock_generate_request_file, mocker): +def test_handle_generate_success( + caplog, mock_generate_request_file, mock_build_bazel_file, mocker +): """ Tests the successful execution path of handle_generate. """ caplog.set_level(logging.INFO) - mock_determine_rule = mocker.patch( - "cli._determine_bazel_rule", return_value="mock-rule" - ) - mock_build_target = mocker.patch("cli._build_bazel_target") - mock_locate_and_extract_artifact = mocker.patch("cli._locate_and_extract_artifact") + mock_generate_api = mocker.patch("cli._generate_api") mock_run_post_processor = mocker.patch("cli._run_post_processor") mock_copy_files_needed_for_post_processing = mocker.patch( "cli._copy_files_needed_for_post_processing" @@ -399,7 +447,6 @@ def test_handle_generate_success(caplog, mock_generate_request_file, mocker): handle_generate() - mock_determine_rule.assert_called_once_with("google/cloud/language/v1", "source") mock_run_post_processor.assert_called_once_with("output", "google-cloud-language") mock_copy_files_needed_for_post_processing.assert_called_once_with( "output", "input", "google-cloud-language" @@ -407,6 +454,7 @@ def test_handle_generate_success(caplog, mock_generate_request_file, mocker): mock_clean_up_files_after_post_processing.assert_called_once_with( "output", "google-cloud-language" ) + mock_generate_api.assert_called_once() def test_handle_generate_fail(caplog):