Skip to content

Commit 81c9ef4

Browse files
author
GitLab CI
committed
Add screenshot_region and screenshot_element to Electron and Android SDKs; fix web test app caching
- Electron SDK: Add screenshot_region (uses capturePage with rect) and screenshot_element (resolves element bounds then captures region) - Android SDK: Add screenshot_region (captures full then crops bitmap) and screenshot_element (finds view bounds then delegates to screenshot_region) - Web test app server: Add no-cache headers to prevent stale SDK serving - Web SDK already had screenshot_region/screenshot_element handlers
1 parent 5d4019c commit 81c9ef4

File tree

3 files changed

+118
-3
lines changed

3 files changed

+118
-3
lines changed

‎sdks/android/src/main/java/com/flutterskill/FlutterSkillBridge.kt‎

Lines changed: 72 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ object FlutterSkillBridge {
6363

6464
// Capabilities advertised in health check
6565
private val capabilities = listOf(
66-
"initialize", "screenshot", "inspect", "inspect_interactive", "tap", "enter_text",
66+
"initialize", "screenshot", "screenshot_region", "screenshot_element", "inspect", "inspect_interactive", "tap", "enter_text",
6767
"swipe", "scroll", "find_element", "get_text", "wait_for_element",
6868
"get_logs", "clear_logs", "go_back", "press_key",
6969
)
@@ -454,6 +454,8 @@ object FlutterSkillBridge {
454454
"get_text" -> handleGetText(params)
455455
"wait_for_element" -> handleWaitForElement(params)
456456
"screenshot" -> handleScreenshot()
457+
"screenshot_region" -> handleScreenshotRegion(params)
458+
"screenshot_element" -> handleScreenshotElement(params)
457459
"get_logs" -> handleGetLogs()
458460
"clear_logs" -> handleClearLogs()
459461
"go_back" -> handleGoBack()
@@ -988,6 +990,75 @@ object FlutterSkillBridge {
988990
return Base64.encodeToString(stream.toByteArray(), Base64.NO_WRAP)
989991
}
990992

993+
private fun handleScreenshotRegion(params: JSONObject): JSONObject {
994+
val screenshotResult = handleScreenshot()
995+
if (!screenshotResult.optBoolean("success", false)) return screenshotResult
996+
997+
val fullBase64 = screenshotResult.optString("image", "")
998+
if (fullBase64.isEmpty()) return screenshotResult
999+
1000+
val x = params.optInt("x", 0)
1001+
val y = params.optInt("y", 0)
1002+
val width = params.optInt("width", 300)
1003+
val height = params.optInt("height", 300)
1004+
1005+
return try {
1006+
val bytes = Base64.decode(fullBase64, Base64.NO_WRAP)
1007+
val fullBitmap = android.graphics.BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
1008+
// Clamp to bitmap bounds
1009+
val cx = x.coerceIn(0, fullBitmap.width - 1)
1010+
val cy = y.coerceIn(0, fullBitmap.height - 1)
1011+
val cw = width.coerceAtMost(fullBitmap.width - cx)
1012+
val ch = height.coerceAtMost(fullBitmap.height - cy)
1013+
val cropped = Bitmap.createBitmap(fullBitmap, cx, cy, cw, ch)
1014+
fullBitmap.recycle()
1015+
val b64 = bitmapToBase64(cropped)
1016+
JSONObject().apply {
1017+
put("success", true)
1018+
put("image", b64)
1019+
put("format", "jpeg")
1020+
put("encoding", "base64")
1021+
}
1022+
} catch (e: Exception) {
1023+
JSONObject().apply {
1024+
put("success", false)
1025+
put("message", "Failed to crop screenshot: ${e.message}")
1026+
}
1027+
}
1028+
}
1029+
1030+
private fun handleScreenshotElement(params: JSONObject): JSONObject {
1031+
val key = params.optString("key", "")
1032+
val refId = params.optString("ref", "")
1033+
1034+
if (key.isEmpty() && refId.isEmpty()) {
1035+
return JSONObject().apply { put("success", false); put("message", "Missing key or ref") }
1036+
}
1037+
1038+
val activity = currentActivity
1039+
?: return JSONObject().apply { put("success", false); put("message", "No active activity") }
1040+
1041+
val bounds = runOnMainThreadBlocking {
1042+
val root = activity.window.decorView.rootView
1043+
var view: View? = null
1044+
if (refId.isNotEmpty()) view = findViewByRefId(root, refId)
1045+
if (view == null && key.isNotEmpty()) view = ViewTraversal.findByKey(root, key)
1046+
if (view != null) {
1047+
val location = IntArray(2)
1048+
view.getLocationOnScreen(location)
1049+
intArrayOf(location[0], location[1], view.width, view.height)
1050+
} else null
1051+
} ?: return JSONObject().apply { put("success", false); put("message", "Element not found") }
1052+
1053+
val regionParams = JSONObject().apply {
1054+
put("x", bounds[0])
1055+
put("y", bounds[1])
1056+
put("width", bounds[2])
1057+
put("height", bounds[3])
1058+
}
1059+
return handleScreenshotRegion(regionParams)
1060+
}
1061+
9911062
private fun handleGetLogs(): JSONObject {
9921063
val logsArray = JSONArray()
9931064
for (entry in logBuffer) {

‎sdks/electron/flutter-skill-electron.js‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ class FlutterSkillElectron {
3030
capabilities: [
3131
'initialize', 'inspect', 'inspect_interactive', 'tap', 'enter_text', 'get_text',
3232
'find_element', 'wait_for_element', 'scroll', 'swipe',
33-
'screenshot', 'go_back', 'get_logs', 'clear_logs', 'press_key',
33+
'screenshot', 'screenshot_region', 'screenshot_element', 'go_back', 'get_logs', 'clear_logs', 'press_key',
3434
],
3535
}));
3636
} else {
@@ -281,6 +281,12 @@ class FlutterSkillElectron {
281281
case 'screenshot':
282282
return this._screenshot(win);
283283

284+
case 'screenshot_region':
285+
return this._screenshotRegion(win, params);
286+
287+
case 'screenshot_element':
288+
return this._screenshotElement(win, params);
289+
284290
case 'go_back':
285291
return this._goBack(win);
286292

@@ -790,6 +796,39 @@ class FlutterSkillElectron {
790796
return { success: true, image: base64, format: 'png', encoding: 'base64' };
791797
}
792798

799+
async _screenshotRegion(win, params) {
800+
if (!win) return { success: false, message: 'No window' };
801+
const x = Math.round(params.x || 0);
802+
const y = Math.round(params.y || 0);
803+
const width = Math.round(params.width || 300);
804+
const height = Math.round(params.height || 300);
805+
const image = await win.webContents.capturePage({ x, y, width, height });
806+
const base64 = image.toPNG().toString('base64');
807+
return { success: true, image: base64, format: 'png', encoding: 'base64' };
808+
}
809+
810+
async _screenshotElement(win, params) {
811+
if (!win) return { success: false, message: 'No window' };
812+
const sel = this._resolveSelector(params);
813+
const refId = params.ref;
814+
815+
const bounds = await win.webContents.executeJavaScript(`
816+
(function() {
817+
${refId ? this._getRefResolutionScript() : ''}
818+
let el = ${refId ? `findElementByRef(${JSON.stringify(refId)})` : 'null'};
819+
if (!el && ${JSON.stringify(sel)}) el = document.querySelector(${JSON.stringify(sel || '')});
820+
if (!el) return null;
821+
const rect = el.getBoundingClientRect();
822+
return { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) };
823+
})();
824+
`);
825+
826+
if (!bounds) return { success: false, message: 'Element not found' };
827+
const image = await win.webContents.capturePage(bounds);
828+
const base64 = image.toPNG().toString('base64');
829+
return { success: true, image: base64, format: 'png', encoding: 'base64' };
830+
}
831+
793832
async _pressKey(win, params) {
794833
if (!win) return { success: false, error: 'No window' };
795834
const keyName = params.key;

‎test/e2e/web_test_app/server.js‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,12 @@ http.createServer(function (req, res) {
2828
res.end('Not Found');
2929
return;
3030
}
31-
res.writeHead(200, { 'Content-Type': MIME[ext] || 'application/octet-stream' });
31+
res.writeHead(200, {
32+
'Content-Type': MIME[ext] || 'application/octet-stream',
33+
'Cache-Control': 'no-store, no-cache, must-revalidate',
34+
'Pragma': 'no-cache',
35+
'Expires': '0'
36+
});
3237
res.end(data);
3338
});
3439
}).listen(PORT, function () {

0 commit comments

Comments
 (0)