|
| 1 | +#!/usr/bin/env python3 |
| 2 | +""" |
| 3 | +Comprehensive validation tests for LLDB-DAP core performance fixes. |
| 4 | +This validates that the core fixes work reliably across different scenarios. |
| 5 | +""" |
| 6 | + |
| 7 | +import subprocess |
| 8 | +import time |
| 9 | +import json |
| 10 | +import os |
| 11 | +import sys |
| 12 | +import tempfile |
| 13 | +from pathlib import Path |
| 14 | + |
| 15 | +class CoreFixValidator: |
| 16 | + def __init__(self, lldb_dap_path): |
| 17 | + self.lldb_dap_path = lldb_dap_path |
| 18 | + self.test_results = {} |
| 19 | + |
| 20 | + def create_test_program(self, name="test_program"): |
| 21 | + """Create a test program with debug info.""" |
| 22 | + test_file = Path(f"{name}.c") |
| 23 | + test_file.write_text(f""" |
| 24 | +#include <stdio.h> |
| 25 | +#include <unistd.h> |
| 26 | +
|
| 27 | +int main() {{ |
| 28 | + printf("Hello from {name}\\n"); |
| 29 | + sleep(1); // Give time for debugger to attach |
| 30 | + return 0; |
| 31 | +}} |
| 32 | +""") |
| 33 | + |
| 34 | + # Compile with debug info |
| 35 | + subprocess.run(["clang", "-g", "-o", name, str(test_file)], check=True) |
| 36 | + return Path(name).absolute() |
| 37 | + |
| 38 | + def test_performance_regression(self): |
| 39 | + """Test that launch times are under 500ms consistently.""" |
| 40 | + print("=== Testing Performance Regression ===") |
| 41 | + |
| 42 | + program = self.create_test_program("perf_test") |
| 43 | + times = [] |
| 44 | + |
| 45 | + for i in range(5): |
| 46 | + start_time = time.time() |
| 47 | + |
| 48 | + try: |
| 49 | + process = subprocess.Popen( |
| 50 | + [str(self.lldb_dap_path)], |
| 51 | + stdin=subprocess.PIPE, |
| 52 | + stdout=subprocess.PIPE, |
| 53 | + stderr=subprocess.PIPE, |
| 54 | + text=True |
| 55 | + ) |
| 56 | + |
| 57 | + # Send minimal DAP sequence |
| 58 | + init_msg = self._create_dap_message("initialize", |
| 59 | + {"clientID": "test"}) |
| 60 | + launch_msg = self._create_dap_message("launch", { |
| 61 | + "program": str(program), |
| 62 | + "stopOnEntry": True |
| 63 | + }) |
| 64 | + |
| 65 | + process.stdin.write(init_msg) |
| 66 | + process.stdin.write(launch_msg) |
| 67 | + process.stdin.flush() |
| 68 | + |
| 69 | + # Wait for response or timeout |
| 70 | + stdout, stderr = process.communicate(timeout=5) |
| 71 | + end_time = time.time() |
| 72 | + |
| 73 | + duration = (end_time - start_time) * 1000 |
| 74 | + times.append(duration) |
| 75 | + print(f" Run {i+1}: {duration:.1f}ms") |
| 76 | + |
| 77 | + except subprocess.TimeoutExpired: |
| 78 | + process.kill() |
| 79 | + times.append(5000) # Timeout |
| 80 | + print(f" Run {i+1}: TIMEOUT") |
| 81 | + |
| 82 | + avg_time = sum(times) / len(times) |
| 83 | + max_time = max(times) |
| 84 | + |
| 85 | + # Validate performance requirements |
| 86 | + performance_ok = avg_time < 500 and max_time < 1000 |
| 87 | + |
| 88 | + self.test_results['performance_regression'] = { |
| 89 | + 'passed': performance_ok, |
| 90 | + 'avg_time_ms': avg_time, |
| 91 | + 'max_time_ms': max_time, |
| 92 | + 'times': times, |
| 93 | + 'requirement': 'avg < 500ms, max < 1000ms' |
| 94 | + } |
| 95 | + |
| 96 | + print(f" Average: {avg_time:.1f}ms, Max: {max_time:.1f}ms") |
| 97 | + print(f" Result: {'PASS' if performance_ok else 'FAIL'}") |
| 98 | + |
| 99 | + return performance_ok |
| 100 | + |
| 101 | + def test_network_symbol_scenarios(self): |
| 102 | + """Test behavior with different network conditions.""" |
| 103 | + print("=== Testing Network Symbol Scenarios ===") |
| 104 | + |
| 105 | + program = self.create_test_program("network_test") |
| 106 | + scenarios = [ |
| 107 | + ("no_debuginfod", {}), |
| 108 | + ("with_debuginfod", |
| 109 | + {"DEBUGINFOD_URLS": "http://debuginfod.example.com"}), |
| 110 | + ("slow_debuginfod", |
| 111 | + {"DEBUGINFOD_URLS": "http://slow.debuginfod.example.com"}), |
| 112 | + ] |
| 113 | + |
| 114 | + results = {} |
| 115 | + |
| 116 | + for scenario_name, env_vars in scenarios: |
| 117 | + print(f" Testing {scenario_name}...") |
| 118 | + |
| 119 | + # Set up environment |
| 120 | + test_env = os.environ.copy() |
| 121 | + test_env.update(env_vars) |
| 122 | + |
| 123 | + start_time = time.time() |
| 124 | + |
| 125 | + try: |
| 126 | + process = subprocess.Popen( |
| 127 | + [str(self.lldb_dap_path)], |
| 128 | + stdin=subprocess.PIPE, |
| 129 | + stdout=subprocess.PIPE, |
| 130 | + stderr=subprocess.PIPE, |
| 131 | + text=True, |
| 132 | + env=test_env |
| 133 | + ) |
| 134 | + |
| 135 | + init_msg = self._create_dap_message("initialize", {"clientID": "test"}) |
| 136 | + launch_msg = self._create_dap_message("launch", { |
| 137 | + "program": str(program), |
| 138 | + "stopOnEntry": True |
| 139 | + }) |
| 140 | + |
| 141 | + process.stdin.write(init_msg) |
| 142 | + process.stdin.write(launch_msg) |
| 143 | + process.stdin.flush() |
| 144 | + |
| 145 | + stdout, stderr = process.communicate(timeout=10) |
| 146 | + end_time = time.time() |
| 147 | + |
| 148 | + duration = (end_time - start_time) * 1000 |
| 149 | + results[scenario_name] = { |
| 150 | + 'duration_ms': duration, |
| 151 | + 'success': True, |
| 152 | + 'timeout': False |
| 153 | + } |
| 154 | + |
| 155 | + print(f" {scenario_name}: {duration:.1f}ms - SUCCESS") |
| 156 | + |
| 157 | + except subprocess.TimeoutExpired: |
| 158 | + process.kill() |
| 159 | + results[scenario_name] = { |
| 160 | + 'duration_ms': 10000, |
| 161 | + 'success': False, |
| 162 | + 'timeout': True |
| 163 | + } |
| 164 | + print(f" {scenario_name}: TIMEOUT - FAIL") |
| 165 | + |
| 166 | + # Validate that all scenarios complete reasonably quickly |
| 167 | + all_passed = all(r['duration_ms'] < 3000 for r in results.values()) |
| 168 | + |
| 169 | + self.test_results['network_scenarios'] = { |
| 170 | + 'passed': all_passed, |
| 171 | + 'scenarios': results |
| 172 | + } |
| 173 | + |
| 174 | + print(f" Overall: {'PASS' if all_passed else 'FAIL'}") |
| 175 | + return all_passed |
| 176 | + |
| 177 | + def test_cross_platform_performance(self): |
| 178 | + """Test performance consistency across different conditions.""" |
| 179 | + print("=== Testing Cross-Platform Performance ===") |
| 180 | + |
| 181 | + # Test with different program sizes |
| 182 | + test_cases = [ |
| 183 | + ("small", self._create_small_program), |
| 184 | + ("medium", self._create_medium_program), |
| 185 | + ] |
| 186 | + |
| 187 | + results = {} |
| 188 | + |
| 189 | + for case_name, program_creator in test_cases: |
| 190 | + print(f" Testing {case_name} program...") |
| 191 | + |
| 192 | + program = program_creator() |
| 193 | + times = [] |
| 194 | + |
| 195 | + for i in range(3): |
| 196 | + start_time = time.time() |
| 197 | + |
| 198 | + try: |
| 199 | + process = subprocess.Popen( |
| 200 | + [str(self.lldb_dap_path)], |
| 201 | + stdin=subprocess.PIPE, |
| 202 | + stdout=subprocess.PIPE, |
| 203 | + stderr=subprocess.PIPE, |
| 204 | + text=True |
| 205 | + ) |
| 206 | + |
| 207 | + init_msg = self._create_dap_message("initialize", {"clientID": "test"}) |
| 208 | + launch_msg = self._create_dap_message("launch", { |
| 209 | + "program": str(program), |
| 210 | + "stopOnEntry": True |
| 211 | + }) |
| 212 | + |
| 213 | + process.stdin.write(init_msg) |
| 214 | + process.stdin.write(launch_msg) |
| 215 | + process.stdin.flush() |
| 216 | + |
| 217 | + stdout, stderr = process.communicate(timeout=5) |
| 218 | + end_time = time.time() |
| 219 | + |
| 220 | + duration = (end_time - start_time) * 1000 |
| 221 | + times.append(duration) |
| 222 | + |
| 223 | + except subprocess.TimeoutExpired: |
| 224 | + process.kill() |
| 225 | + times.append(5000) |
| 226 | + |
| 227 | + avg_time = sum(times) / len(times) |
| 228 | + results[case_name] = { |
| 229 | + 'avg_time_ms': avg_time, |
| 230 | + 'times': times, |
| 231 | + 'passed': avg_time < 1000 |
| 232 | + } |
| 233 | + |
| 234 | + print(f" {case_name}: {avg_time:.1f}ms avg - {'PASS' if avg_time < 1000 else 'FAIL'}") |
| 235 | + |
| 236 | + all_passed = all(r['passed'] for r in results.values()) |
| 237 | + |
| 238 | + self.test_results['cross_platform'] = { |
| 239 | + 'passed': all_passed, |
| 240 | + 'cases': results |
| 241 | + } |
| 242 | + |
| 243 | + return all_passed |
| 244 | + |
| 245 | + def _create_small_program(self): |
| 246 | + """Create a small test program.""" |
| 247 | + return self.create_test_program("small_test") |
| 248 | + |
| 249 | + def _create_medium_program(self): |
| 250 | + """Create a medium-sized test program with more symbols.""" |
| 251 | + test_file = Path("medium_test.c") |
| 252 | + test_file.write_text(""" |
| 253 | +#include <stdio.h> |
| 254 | +#include <stdlib.h> |
| 255 | +#include <string.h> |
| 256 | +
|
| 257 | +struct TestStruct { |
| 258 | + int value; |
| 259 | + char name[64]; |
| 260 | + double data[100]; |
| 261 | +}; |
| 262 | +
|
| 263 | +void function1() { printf("Function 1\\n"); } |
| 264 | +void function2() { printf("Function 2\\n"); } |
| 265 | +void function3() { printf("Function 3\\n"); } |
| 266 | +
|
| 267 | +int main() { |
| 268 | + struct TestStruct test; |
| 269 | + test.value = 42; |
| 270 | + strcpy(test.name, "test"); |
| 271 | + |
| 272 | + for (int i = 0; i < 100; i++) { |
| 273 | + test.data[i] = i * 3.14; |
| 274 | + } |
| 275 | + |
| 276 | + function1(); |
| 277 | + function2(); |
| 278 | + function3(); |
| 279 | + |
| 280 | + return 0; |
| 281 | +} |
| 282 | +""") |
| 283 | + |
| 284 | + subprocess.run(["clang", "-g", "-o", "medium_test", str(test_file)], check=True) |
| 285 | + return Path("medium_test").absolute() |
| 286 | + |
| 287 | + def _create_dap_message(self, command, arguments=None): |
| 288 | + """Create a DAP protocol message.""" |
| 289 | + if arguments is None: |
| 290 | + arguments = {} |
| 291 | + |
| 292 | + message = { |
| 293 | + "seq": 1, |
| 294 | + "type": "request", |
| 295 | + "command": command, |
| 296 | + "arguments": arguments |
| 297 | + } |
| 298 | + |
| 299 | + content = json.dumps(message) |
| 300 | + return f"Content-Length: {len(content)}\r\n\r\n{content}" |
| 301 | + |
| 302 | + def run_all_tests(self): |
| 303 | + """Run all validation tests.""" |
| 304 | + print("LLDB-DAP Core Fix Validation") |
| 305 | + print("=" * 50) |
| 306 | + |
| 307 | + tests = [ |
| 308 | + ("Performance Regression", self.test_performance_regression), |
| 309 | + ("Network Symbol Scenarios", self.test_network_symbol_scenarios), |
| 310 | + ("Cross-Platform Performance", self.test_cross_platform_performance), |
| 311 | + ] |
| 312 | + |
| 313 | + passed_tests = 0 |
| 314 | + total_tests = len(tests) |
| 315 | + |
| 316 | + for test_name, test_func in tests: |
| 317 | + try: |
| 318 | + if test_func(): |
| 319 | + passed_tests += 1 |
| 320 | + print() |
| 321 | + except Exception as e: |
| 322 | + print(f" ERROR: {e}") |
| 323 | + print() |
| 324 | + |
| 325 | + # Summary |
| 326 | + print("=" * 50) |
| 327 | + print("VALIDATION SUMMARY:") |
| 328 | + print("=" * 50) |
| 329 | + print(f"Tests passed: {passed_tests}/{total_tests}") |
| 330 | + |
| 331 | + for test_name, result in self.test_results.items(): |
| 332 | + status = "PASS" if result['passed'] else "FAIL" |
| 333 | + print(f"{test_name:25}: {status}") |
| 334 | + |
| 335 | + overall_success = passed_tests == total_tests |
| 336 | + print(f"\nOverall result: {'SUCCESS' if overall_success else 'FAILURE'}") |
| 337 | + |
| 338 | + return overall_success |
| 339 | + |
| 340 | +def main(): |
| 341 | + """Main validation function.""" |
| 342 | + lldb_dap_path = Path("./build/bin/lldb-dap") |
| 343 | + |
| 344 | + if not lldb_dap_path.exists(): |
| 345 | + print(f"Error: lldb-dap not found at {lldb_dap_path}") |
| 346 | + return 1 |
| 347 | + |
| 348 | + validator = CoreFixValidator(lldb_dap_path) |
| 349 | + success = validator.run_all_tests() |
| 350 | + |
| 351 | + return 0 if success else 1 |
| 352 | + |
| 353 | +if __name__ == "__main__": |
| 354 | + sys.exit(main()) |
0 commit comments