diff --git a/.generator/Dockerfile b/.generator/Dockerfile index ee96e705b0aa..0d98536ab42a 100644 --- a/.generator/Dockerfile +++ b/.generator/Dockerfile @@ -36,20 +36,37 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* # Set up environment variables for tool versions to make updates easier. -ENV PYTHON_VERSION=3.11.5 ENV BAZELISK_VERSION=v1.26.0 -# Create a symbolic link for `python3` to point to our specific version. -ENV PATH /usr/local/bin/python3.11:$PATH - -# Install Python from source -RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz && \ +RUN for PYTHON_VERSION in 3.9.23 3.10.18 3.11.13 3.12.11 3.13.5; do \ + # Install Python from source + wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz && \ tar -xvf Python-${PYTHON_VERSION}.tgz && \ cd Python-${PYTHON_VERSION} && \ ./configure --enable-optimizations && \ make altinstall && \ cd / && \ - rm -rf Python-${PYTHON_VERSION}* + rm -rf Python-${PYTHON_VERSION}* \ + ; done + +# Get the pip installation script using the instructions below +# https://pip.pypa.io/en/stable/installation/#get-pip-py +# Attempts to use `ensurepip` instead of `get-pip.py` resulted in errors like `python3.13.5 not found` +RUN wget --no-check-certificate -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' + +# Install pip for each python version +RUN for PYTHON_VERSION in 3.9 3.10 3.11 3.12 3.13; do \ + python${PYTHON_VERSION} /tmp/get-pip.py \ + ; done + +# Remove the pip installation script +RUN rm /tmp/get-pip.py + +# Test Pip +RUN for PYTHON_VERSION in 3.9 3.10 3.11 3.12 3.13; do \ + python${PYTHON_VERSION} -m pip \ + ; done + # TODO(https://github.com/googleapis/librarian/issues/904): Install protoc for gencode. diff --git a/.generator/cli.py b/.generator/cli.py index adc3cc4c242c..2d216d0aac3a 100644 --- a/.generator/cli.py +++ b/.generator/cli.py @@ -16,8 +16,10 @@ import json import logging import os +import subprocess import sys import subprocess +from typing import Dict, List logger = logging.getLogger() @@ -25,9 +27,10 @@ GENERATE_REQUEST_FILE = "generate-request.json" SOURCE_DIR = "source" OUTPUT_DIR = "output" +REPO_DIR = "repo" -def _read_json_file(path): +def _read_json_file(path: str) -> Dict: """Helper function that reads a json file path and returns the loaded json content. Args: @@ -50,7 +53,7 @@ def handle_configure(): logger.info("'configure' command executed.") -def _determine_bazel_rule(api_path): +def _determine_bazel_rule(api_path: str) -> str: """Executes a `bazelisk query` to find a Bazel rule. Args: @@ -83,7 +86,25 @@ def _determine_bazel_rule(api_path): raise ValueError(f"Bazelisk query `{query}` failed") from e -def _build_bazel_target(bazel_rule): +def _get_library_id(request_data: Dict) -> str: + """Retrieve the library id from the given request dictionary + + Args: + request_data(Dict): The contents `generate-request.json`. + + Raises: + ValueError: If the key `id` does not exist in `request_data`. + + Returns: + str: The id of the library in `generate-request.json` + """ + library_id = request_data.get("id") + if not library_id: + raise ValueError("Request file is missing required 'id' field.") + return library_id + + +def _build_bazel_target(bazel_rule: str): """Executes `bazelisk build` on a given Bazel rule. Args: @@ -167,9 +188,7 @@ def handle_generate(): # Read a generate-request.json file try: request_data = _read_json_file(f"{LIBRARIAN_DIR}/{GENERATE_REQUEST_FILE}") - library_id = request_data.get("id") - if not library_id: - raise ValueError("Request file is missing required 'id' field.") + library_id = _get_library_id(request_data) for api in request_data.get("apis", []): api_path = api.get("path") @@ -186,8 +205,59 @@ def handle_generate(): logger.info("'generate' command executed.") +def _run_nox_sessions(sessions: List[str]): + """Calls nox for all specified sessions. + + Args: + path(List[str]): The list of nox sessions to run. + """ + # Read a generate-request.json file + current_session = None + try: + request_data = _read_json_file(f"{LIBRARIAN_DIR}/{GENERATE_REQUEST_FILE}") + library_id = _get_library_id(request_data) + for nox_session in sessions: + _run_individual_session(nox_session, library_id) + except Exception as e: + raise ValueError(f"Failed to run the nox session: {current_session}") from e + + +def _run_individual_session(nox_session: str, library_id: str): + """ + Calls nox with the specified sessions. + + Args: + nox_session(str): The nox session to run + library_id(str): The library id under test + """ + command = [ + "nox", + "-s", + nox_session, + "-f", + f"{REPO_DIR}/packages/{library_id}", + ] + result = subprocess.run(command, text=True, check=True) + logger.info(result) + + def handle_build(): - # TODO(https://github.com/googleapis/librarian/issues/450): Implement build command and update docstring. + """The main coordinator for validating client library generation.""" + sessions = [ + "unit-3.9", + "unit-3.10", + "unit-3.11", + "unit-3.12", + "unit-3.13", + "docs", + "system", + "lint", + "lint_setup_py", + "mypy", + "check_lower_bounds", + ] + _run_nox_sessions(sessions) + logger.info("'build' command executed.") diff --git a/.generator/test_cli.py b/.generator/test_cli.py index f2c4ebe4e706..d14619443a72 100644 --- a/.generator/test_cli.py +++ b/.generator/test_cli.py @@ -12,24 +12,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import pytest import json import logging +import os import subprocess +from unittest.mock import MagicMock, mock_open -from unittest.mock import mock_open, MagicMock +import pytest from cli import ( - _read_json_file, - _determine_bazel_rule, + GENERATE_REQUEST_FILE, + LIBRARIAN_DIR, + REPO_DIR, _build_bazel_target, + _determine_bazel_rule, + _get_library_id, _locate_and_extract_artifact, - handle_generate, + _read_json_file, + _run_individual_session, + _run_nox_sessions, handle_build, handle_configure, - LIBRARIAN_DIR, - GENERATE_REQUEST_FILE, + handle_generate, ) @@ -53,6 +57,42 @@ def mock_generate_request_file(tmp_path, monkeypatch): return request_file +@pytest.fixture +def mock_generate_request_data_for_nox(): + """Returns mock data for generate-request.json for nox tests.""" + return { + "id": "mock-library", + "apis": [ + {"path": "google/mock/v1"}, + ], + } + + +def test_get_library_id_success(): + """Tests that _get_library_id returns the correct ID when present.""" + request_data = {"id": "test-library", "name": "Test Library"} + library_id = _get_library_id(request_data) + assert library_id == "test-library" + + +def test_get_library_id_missing_id(): + """Tests that _get_library_id raises ValueError when 'id' is missing.""" + request_data = {"name": "Test Library"} + with pytest.raises( + ValueError, match="Request file is missing required 'id' field." + ): + _get_library_id(request_data) + + +def test_get_library_id_empty_id(): + """Tests that _get_library_id raises ValueError when 'id' is an empty string.""" + request_data = {"id": "", "name": "Test Library"} + with pytest.raises( + ValueError, match="Request file is missing required 'id' field." + ): + _get_library_id(request_data) + + def test_handle_configure_success(caplog, mock_generate_request_file): """ Tests the successful execution path of handle_configure. @@ -191,12 +231,105 @@ def test_handle_generate_fail(caplog): handle_generate() -def test_handle_build_success(caplog, mock_generate_request_file): +def test_run_individual_session_success(mocker, caplog): + """Tests that _run_individual_session calls nox with correct arguments and logs success.""" + caplog.set_level(logging.INFO) + + mock_subprocess_run = mocker.patch( + "cli.subprocess.run", return_value=MagicMock(returncode=0) + ) + + test_session = "unit-3.9" + test_library_id = "test-library" + _run_individual_session(test_session, test_library_id) + + expected_command = [ + "nox", + "-s", + test_session, + "-f", + f"{REPO_DIR}/packages/{test_library_id}", + ] + mock_subprocess_run.assert_called_once_with(expected_command, text=True, check=True) + + +def test_run_individual_session_failure(mocker): + """Tests that _run_individual_session raises CalledProcessError if nox command fails.""" + mocker.patch( + "cli.subprocess.run", + side_effect=subprocess.CalledProcessError( + 1, "nox", stderr="Nox session failed" + ), + ) + + with pytest.raises(subprocess.CalledProcessError): + _run_individual_session("lint", "another-library") + + +def test_run_nox_sessions_success(mocker, mock_generate_request_data_for_nox): + """Tests that _run_nox_sessions successfully runs all specified sessions.""" + mocker.patch("cli._read_json_file", return_value=mock_generate_request_data_for_nox) + mocker.patch("cli._get_library_id", return_value="mock-library") + mock_run_individual_session = mocker.patch("cli._run_individual_session") + + sessions_to_run = ["unit-3.9", "lint"] + _run_nox_sessions(sessions_to_run) + + assert mock_run_individual_session.call_count == len(sessions_to_run) + mock_run_individual_session.assert_has_calls( + [ + mocker.call("unit-3.9", "mock-library"), + mocker.call("lint", "mock-library"), + ] + ) + + +def test_run_nox_sessions_read_file_failure(mocker): + """Tests that _run_nox_sessions raises ValueError if _read_json_file fails.""" + mocker.patch("cli._read_json_file", side_effect=FileNotFoundError("file not found")) + + with pytest.raises(ValueError, match="Failed to run the nox session"): + _run_nox_sessions(["unit-3.9"]) + + +def test_run_nox_sessions_get_library_id_failure(mocker): + """Tests that _run_nox_sessions raises ValueError if _get_library_id fails.""" + mocker.patch("cli._read_json_file", return_value={"apis": []}) # Missing 'id' + mocker.patch( + "cli._get_library_id", + side_effect=ValueError("Request file is missing required 'id' field."), + ) + + with pytest.raises(ValueError, match="Failed to run the nox session"): + _run_nox_sessions(["unit-3.9"]) + + +def test_run_nox_sessions_individual_session_failure( + mocker, mock_generate_request_data_for_nox +): + """Tests that _run_nox_sessions raises ValueError if _run_individual_session fails.""" + mocker.patch("cli._read_json_file", return_value=mock_generate_request_data_for_nox) + mocker.patch("cli._get_library_id", return_value="mock-library") + mock_run_individual_session = mocker.patch( + "cli._run_individual_session", + side_effect=[None, subprocess.CalledProcessError(1, "nox", "session failed")], + ) + + sessions_to_run = ["unit-3.9", "lint"] + with pytest.raises(ValueError, match="Failed to run the nox session"): + _run_nox_sessions(sessions_to_run) + + # Check that _run_individual_session was called at least once + assert mock_run_individual_session.call_count > 0 + + +def test_handle_build_success(caplog, mocker): """ Tests the successful execution path of handle_build. """ caplog.set_level(logging.INFO) + mocker.patch("cli._run_nox_sessions") handle_build() assert "'build' command executed." in caplog.text diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 2a7dd688e1af..6c3dba47ec10 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# TODO(https://github.com/googleapis/google-cloud-python/issues/14142): +# Reduce this timeout by moving the installation of Python runtimes to a separate base image +timeout: 7200s # 2 hours steps: # This step builds the Docker image. - name: 'gcr.io/cloud-builders/docker'