Skip to content

Commit 3462f2d

Browse files
author
GitLab CI
committed
feat: visual_verify and visual_diff tools for AI-powered UI verification
1 parent 7463ac5 commit 3462f2d

File tree

1 file changed

+115
-0
lines changed

1 file changed

+115
-0
lines changed

lib/src/cli/server.dart

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,9 +81,13 @@ Future<void> runServer(List<String> args) async {
8181
autoUrl = arg.substring('--url='.length);
8282
} else if (arg.startsWith('--cdp-port=')) {
8383
cdpPort = int.tryParse(arg.substring('--cdp-port='.length));
84+
} else if (arg.startsWith('--plugins-dir=')) {
85+
server._pluginsDir = arg.substring('--plugins-dir='.length);
8486
}
8587
}
8688

89+
await server._loadPlugins();
90+
8791
if (autoUrl != null) {
8892
server._autoConnectUrl = autoUrl;
8993
server._autoConnectCdpPort = cdpPort;
@@ -4274,6 +4278,117 @@ can visually compare them. Also returns text snapshots for structural comparison
42744278
"Returning base64 data. Consider using save_to_file=true for large regions."
42754279
};
42764280

4281+
// AI Visual Verification
4282+
case 'visual_verify':
4283+
final verifyQuality = (args['quality'] as num?)?.toDouble() ?? 0.5;
4284+
final verifyDesc = args['description'] as String? ?? '';
4285+
final checkElements = (args['check_elements'] as List?)?.cast<String>() ?? [];
4286+
4287+
// Take screenshot
4288+
final verifyImageBase64 = await client!.takeScreenshot(quality: verifyQuality, maxWidth: 800);
4289+
String? verifyScreenshotPath;
4290+
if (verifyImageBase64 != null) {
4291+
final tempDir = Directory.systemTemp;
4292+
final timestamp = DateTime.now().millisecondsSinceEpoch;
4293+
final file = File('${tempDir.path}/flutter_skill_verify_$timestamp.png');
4294+
await file.writeAsBytes(base64.decode(verifyImageBase64));
4295+
verifyScreenshotPath = file.path;
4296+
}
4297+
4298+
// Take snapshot (text tree)
4299+
String verifySnapshotText = '';
4300+
List<String> foundElements = [];
4301+
List<String> missingElements = [];
4302+
int verifyElementCount = 0;
4303+
try {
4304+
final structured = await client!.getInteractiveElementsStructured();
4305+
final snapshotElements = structured['elements'] as List<dynamic>? ?? [];
4306+
verifyElementCount = snapshotElements.length;
4307+
4308+
final buf = StringBuffer();
4309+
for (var i = 0; i < snapshotElements.length; i++) {
4310+
final el = snapshotElements[i] as Map<String, dynamic>;
4311+
final ref = el['ref'] ?? '';
4312+
final text = el['text']?.toString() ?? '';
4313+
final label = el['label']?.toString() ?? '';
4314+
final display = text.isNotEmpty ? text : label;
4315+
buf.writeln('[$ref] "$display"');
4316+
}
4317+
verifySnapshotText = buf.toString();
4318+
4319+
// Check elements
4320+
if (checkElements.isNotEmpty) {
4321+
final snapshotLower = verifySnapshotText.toLowerCase();
4322+
for (final check in checkElements) {
4323+
if (snapshotLower.contains(check.toLowerCase())) {
4324+
foundElements.add(check);
4325+
} else {
4326+
missingElements.add(check);
4327+
}
4328+
}
4329+
}
4330+
} catch (e) {
4331+
verifySnapshotText = 'Error getting snapshot: $e';
4332+
}
4333+
4334+
return {
4335+
'success': true,
4336+
'screenshot': verifyScreenshotPath,
4337+
'snapshot': verifySnapshotText,
4338+
'elements_found': foundElements,
4339+
'elements_missing': missingElements,
4340+
'element_count': verifyElementCount,
4341+
'description_to_verify': verifyDesc,
4342+
'hint': 'Compare the screenshot and snapshot against the description. Report any discrepancies.',
4343+
};
4344+
4345+
case 'visual_diff':
4346+
final diffQuality = (args['quality'] as num?)?.toDouble() ?? 0.5;
4347+
final baselinePath = args['baseline_path'] as String;
4348+
final diffDesc = args['description'] as String? ?? '';
4349+
4350+
final baselineFile = File(baselinePath);
4351+
if (!await baselineFile.exists()) {
4352+
return {'success': false, 'error': 'Baseline file not found: $baselinePath'};
4353+
}
4354+
4355+
final diffImageBase64 = await client!.takeScreenshot(quality: diffQuality, maxWidth: 800);
4356+
String? currentScreenshotPath;
4357+
if (diffImageBase64 != null) {
4358+
final tempDir = Directory.systemTemp;
4359+
final timestamp = DateTime.now().millisecondsSinceEpoch;
4360+
final file = File('${tempDir.path}/flutter_skill_diff_$timestamp.png');
4361+
await file.writeAsBytes(base64.decode(diffImageBase64));
4362+
currentScreenshotPath = file.path;
4363+
}
4364+
4365+
String diffSnapshotText = '';
4366+
try {
4367+
final structured = await client!.getInteractiveElementsStructured();
4368+
final els = structured['elements'] as List<dynamic>? ?? [];
4369+
final buf = StringBuffer();
4370+
for (final el in els) {
4371+
if (el is Map<String, dynamic>) {
4372+
final ref = el['ref'] ?? '';
4373+
final text = el['text']?.toString() ?? '';
4374+
final label = el['label']?.toString() ?? '';
4375+
buf.writeln('[$ref] "${text.isNotEmpty ? text : label}"');
4376+
}
4377+
}
4378+
diffSnapshotText = buf.toString();
4379+
} catch (e) {
4380+
diffSnapshotText = 'Error: $e';
4381+
}
4382+
4383+
return {
4384+
'success': true,
4385+
'baseline_path': baselinePath,
4386+
'current_screenshot': currentScreenshotPath,
4387+
'current_snapshot': diffSnapshotText,
4388+
'description': diffDesc,
4389+
'hint': 'Compare the baseline screenshot with the current screenshot. Look for visual differences. The text snapshot shows the current UI structure.',
4390+
};
4391+
42774392
case 'screenshot_element':
42784393
// Support both key and text parameters
42794394
String? targetKey = args['key'];

0 commit comments

Comments
 (0)