|
| 1 | +# Copyright 2025 Google LLC |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | + |
| 15 | +"""CLI commands for ADK conformance testing.""" |
| 16 | + |
| 17 | +from __future__ import annotations |
| 18 | + |
| 19 | +from pathlib import Path |
| 20 | + |
| 21 | +import click |
| 22 | +from google.genai import types |
| 23 | + |
| 24 | +from ...utils.yaml_utils import dump_pydantic_to_yaml |
| 25 | +from ..adk_web_server import RunAgentRequest |
| 26 | +from ._generated_file_utils import load_test_case |
| 27 | +from .adk_web_server_client import AdkWebServerClient |
| 28 | +from .test_case import TestCase |
| 29 | + |
| 30 | + |
| 31 | +async def _create_conformance_test_files( |
| 32 | + test_case: TestCase, |
| 33 | + user_id: str = "adk_conformance_test_user", |
| 34 | +) -> Path: |
| 35 | + """Generate conformance test files from TestCase.""" |
| 36 | + # Clean existing generated files |
| 37 | + test_case_dir = test_case.dir |
| 38 | + |
| 39 | + # Remove existing generated files to ensure clean state |
| 40 | + generated_session_file = test_case_dir / "generated-session.yaml" |
| 41 | + generated_recordings_file = test_case_dir / "generated-recordings.yaml" |
| 42 | + |
| 43 | + generated_session_file.unlink(missing_ok=True) |
| 44 | + generated_recordings_file.unlink(missing_ok=True) |
| 45 | + |
| 46 | + async with AdkWebServerClient() as client: |
| 47 | + # Create a new session for the test |
| 48 | + session = await client.create_session( |
| 49 | + app_name=test_case.test_spec.agent, user_id=user_id, state={} |
| 50 | + ) |
| 51 | + |
| 52 | + # Run the agent with the user messages |
| 53 | + for user_message_index, user_message in enumerate( |
| 54 | + test_case.test_spec.user_messages |
| 55 | + ): |
| 56 | + content = types.Content( |
| 57 | + parts=[types.Part(text=user_message)], role="user" |
| 58 | + ) |
| 59 | + async for _ in client.run_agent( |
| 60 | + RunAgentRequest( |
| 61 | + app_name=test_case.test_spec.agent, |
| 62 | + user_id=user_id, |
| 63 | + session_id=session.id, |
| 64 | + new_message=content, |
| 65 | + ), |
| 66 | + mode="record", |
| 67 | + test_case_dir=str(test_case_dir), |
| 68 | + user_message_index=user_message_index, |
| 69 | + ): |
| 70 | + pass |
| 71 | + |
| 72 | + # Retrieve the updated session |
| 73 | + updated_session = await client.get_session( |
| 74 | + app_name=test_case.test_spec.agent, |
| 75 | + user_id=user_id, |
| 76 | + session_id=session.id, |
| 77 | + ) |
| 78 | + |
| 79 | + # Save session.yaml |
| 80 | + dump_pydantic_to_yaml( |
| 81 | + updated_session, |
| 82 | + generated_session_file, |
| 83 | + indent=2, |
| 84 | + sort_keys=False, |
| 85 | + exclude_none=True, |
| 86 | + ) |
| 87 | + |
| 88 | + return generated_session_file |
| 89 | + |
| 90 | + |
| 91 | +async def run_conformance_create(paths: list[Path]) -> None: |
| 92 | + """Generate conformance tests from TestCaseInput files. |
| 93 | +
|
| 94 | + Args: |
| 95 | + paths: list of directories containing test cases input files (spec.yaml). |
| 96 | + """ |
| 97 | + click.echo("Generating ADK conformance tests...") |
| 98 | + |
| 99 | + # Look for spec.yaml files and load TestCase objects |
| 100 | + test_cases: dict[Path, TestCase] = {} |
| 101 | + |
| 102 | + for test_dir in paths: |
| 103 | + if not test_dir.exists(): |
| 104 | + continue |
| 105 | + |
| 106 | + for spec_file in test_dir.rglob("spec.yaml"): |
| 107 | + try: |
| 108 | + test_case_dir = spec_file.parent |
| 109 | + category = test_case_dir.parent.name |
| 110 | + name = test_case_dir.name |
| 111 | + test_spec = load_test_case(test_case_dir) |
| 112 | + test_case = TestCase( |
| 113 | + category=category, |
| 114 | + name=name, |
| 115 | + dir=test_case_dir, |
| 116 | + test_spec=test_spec, |
| 117 | + ) |
| 118 | + test_cases[test_case_dir] = test_case |
| 119 | + click.echo(f"Loaded test spec: {category}/{name}") |
| 120 | + except Exception as e: |
| 121 | + click.secho(f"Failed to load {spec_file}: {e}", fg="red", err=True) |
| 122 | + |
| 123 | + # Process all loaded test cases |
| 124 | + if test_cases: |
| 125 | + click.echo(f"\nProcessing {len(test_cases)} test cases...") |
| 126 | + |
| 127 | + for test_case in test_cases.values(): |
| 128 | + try: |
| 129 | + await _create_conformance_test_files(test_case) |
| 130 | + click.secho( |
| 131 | + "Generated conformance test files for:" |
| 132 | + f" {test_case.category}/{test_case.name}", |
| 133 | + fg="green", |
| 134 | + ) |
| 135 | + except Exception as e: |
| 136 | + click.secho( |
| 137 | + f"Failed to generate {test_case.category}/{test_case.name}: {e}", |
| 138 | + fg="red", |
| 139 | + err=True, |
| 140 | + ) |
| 141 | + else: |
| 142 | + click.secho("No test specs found to process.", fg="yellow") |
| 143 | + |
| 144 | + click.secho("\nConformance test generation complete!", fg="blue") |
0 commit comments