Skip to content

Commit 1658ea5

Browse files
Suprhimpclaude
Suprhimp
and
claude
authored
Fix MJPEG fallback glitch: serialize pipeline + guard decoder (#44)
* Fix MJPEG fallback glitch: serialize pipeline + guard decoder - Serialize rebuildPipeline with a Mutex and make currentCodecMode @volatile so viewport rebuilds and codec switches can't race into dual encoders (VideoEncoder + JpegEncoder) with a last-write-wins VD surface. - Collapse onCodecModeRequest into a delegate that sets the mode and reruns rebuildPipeline under the lock — single code path for encoder/VD setup. - Client: MJPEG decoder drops non-JPEG payloads (magic-byte check) to avoid InvalidStateError spam during the codec-switch startup window. - Client: for MJPEG mode, open the control socket and wait for it before connecting video, so the server receives the codec preference before streaming starts. - Split drawer: group apps by category (Navigation/Video/Music/Apps) with colored section headers. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> * Extract codec-mode transition guard into testable pure policy The "should a codec-mode request trigger a rebuild?" decision was inline inside MirrorForegroundService.onCodecModeRequest, which is an Android Service and therefore not reachable from JVM unit tests. Moving the three-line guard into a CodecModeTransition pure object lets the decision table be covered directly (non-mjpeg rejected, already-mjpeg-with-encoder no-op, and the stale-mode recovery case where the mode flag is "mjpeg" but the JpegEncoder is missing). Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
1 parent 549c15d commit 1658ea5

6 files changed

Lines changed: 191 additions & 75 deletions

File tree

‎app/src/main/assets/web/index.html‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,30 @@
214214
margin-right: 12px;
215215
object-fit: contain;
216216
}
217+
.split-category-section {
218+
margin-bottom: 16px;
219+
}
220+
.split-category-header {
221+
display: flex;
222+
align-items: center;
223+
padding: 4px 4px 8px 4px;
224+
}
225+
.split-category-bar {
226+
width: 4px;
227+
height: 18px;
228+
border-radius: 2px;
229+
margin-right: 10px;
230+
}
231+
.split-category-title {
232+
font-size: 15px;
233+
font-weight: bold;
234+
color: #ddd;
235+
}
236+
.split-category-items {
237+
display: flex;
238+
flex-direction: column;
239+
gap: 8px;
240+
}
217241

218242
/* ── Bubble Composer ── */
219243
#input-bubble {

‎app/src/main/assets/web/js/fallback.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,12 @@ class FallbackDecoder {
5050
if (view.length < 9) return;
5151
if (view[0] === 0x02) return; // skip SPS/PPS config (not relevant for MJPEG)
5252

53+
// Silently drop non-JPEG payloads. During the startup window, the server
54+
// may briefly emit H.264 frames before it processes the client's
55+
// `codec: mjpeg` control message — those frames would otherwise raise
56+
// `InvalidStateError` in createImageBitmap and spam the console.
57+
if (view[8] !== 0xFF || view[9] !== 0xD8) return;
58+
5359
// Drop frame if previous decode is still running
5460
if (this._decoding) {
5561
this._droppedFrames++;

‎app/src/main/assets/web/js/main.js‎

Lines changed: 76 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1047,6 +1047,18 @@ document.addEventListener('DOMContentLoaded', async () => {
10471047
else resizeTimer = setTimeout(doSend, 500);
10481048
}
10491049

1050+
function waitForControlSocketOpen(timeoutMs) {
1051+
return new Promise((resolve) => {
1052+
const deadline = Date.now() + timeoutMs;
1053+
const check = () => {
1054+
if (controlSocket && controlSocket.readyState === WebSocket.OPEN) return resolve();
1055+
if (Date.now() >= deadline) return resolve(); // best-effort — fall through on timeout
1056+
setTimeout(check, 20);
1057+
};
1058+
check();
1059+
});
1060+
}
1061+
10501062
function connectControl() {
10511063
const wsUrl = `ws://${host}/ws/control`;
10521064
controlSocket = new WebSocket(wsUrl);
@@ -1218,26 +1230,64 @@ document.addEventListener('DOMContentLoaded', async () => {
12181230
if (!splitAppList) return;
12191231
splitAppList.innerHTML = '';
12201232

1233+
const grouped = {
1234+
'NAVIGATION': { title: 'Navigation', color: '#4CAF50', items: [] },
1235+
'VIDEO': { title: 'Video', color: '#FF5722', items: [] },
1236+
'MUSIC': { title: 'Music', color: '#9C27B0', items: [] },
1237+
'OTHER': { title: 'Apps', color: '#9E9E9E', items: [] }
1238+
};
1239+
12211240
apps.forEach(app => {
1222-
const cell = document.createElement('div');
1223-
cell.className = 'split-app-item';
1241+
if (grouped[app.category]) grouped[app.category].items.push(app);
1242+
else grouped['OTHER'].items.push(app);
1243+
});
12241244

1225-
const icon = document.createElement('img');
1226-
icon.className = 'split-app-icon';
1227-
icon.src = `/api/icon?pkg=${app.packageName}`;
1228-
cell.appendChild(icon);
1245+
Object.keys(grouped).forEach(key => {
1246+
const group = grouped[key];
1247+
if (group.items.length === 0) return;
12291248

1230-
const label = document.createElement('div');
1231-
label.textContent = SPLIT_STRATEGY === 'freeform' ? `${app.label} (Split)` : `${app.label} (Dual Stream)`;
1232-
label.style.color = '#FFD700';
1233-
cell.appendChild(label);
1249+
const section = document.createElement('div');
1250+
section.className = 'split-category-section';
12341251

1235-
cell.addEventListener('click', () => {
1236-
launchApp(app, true);
1237-
splitDrawer.classList.remove('open');
1252+
const header = document.createElement('div');
1253+
header.className = 'split-category-header';
1254+
const bar = document.createElement('div');
1255+
bar.className = 'split-category-bar';
1256+
bar.style.backgroundColor = group.color;
1257+
const title = document.createElement('div');
1258+
title.className = 'split-category-title';
1259+
title.textContent = group.title;
1260+
header.appendChild(bar);
1261+
header.appendChild(title);
1262+
section.appendChild(header);
1263+
1264+
const items = document.createElement('div');
1265+
items.className = 'split-category-items';
1266+
1267+
group.items.forEach(app => {
1268+
const cell = document.createElement('div');
1269+
cell.className = 'split-app-item';
1270+
1271+
const icon = document.createElement('img');
1272+
icon.className = 'split-app-icon';
1273+
icon.src = `/api/icon?pkg=${app.packageName}`;
1274+
cell.appendChild(icon);
1275+
1276+
const label = document.createElement('div');
1277+
label.textContent = SPLIT_STRATEGY === 'freeform' ? `${app.label} (Split)` : `${app.label} (Dual Stream)`;
1278+
label.style.color = '#FFD700';
1279+
cell.appendChild(label);
1280+
1281+
cell.addEventListener('click', () => {
1282+
launchApp(app, true);
1283+
splitDrawer.classList.remove('open');
1284+
});
1285+
1286+
items.appendChild(cell);
12381287
});
12391288

1240-
splitAppList.appendChild(cell);
1289+
section.appendChild(items);
1290+
splitAppList.appendChild(section);
12411291
});
12421292
}
12431293

@@ -1557,8 +1607,18 @@ document.addEventListener('DOMContentLoaded', async () => {
15571607

15581608
try {
15591609
await initDecoder();
1560-
connectVideo();
1561-
connectControl();
1610+
if (codecMode === 'mjpeg') {
1611+
// Open the control socket first so the `codec: mjpeg` preference
1612+
// reaches the server before the video socket starts streaming.
1613+
// Otherwise the server ships H.264 until it processes the switch,
1614+
// which an MJPEG decoder can't render.
1615+
connectControl();
1616+
await waitForControlSocketOpen(2000);
1617+
connectVideo();
1618+
} else {
1619+
connectVideo();
1620+
connectControl();
1621+
}
15621622
} catch (e) {
15631623
setStatus(e.message, 'error');
15641624
showOverlay();

‎app/src/main/java/com/castla/mirror/policy/CodecModeTransition.kt‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package com.castla.mirror.policy
2+
3+
/**
4+
* Pure decision for whether a client codec-mode request should trigger a
5+
* pipeline rebuild.
6+
*
7+
* Encapsulates the guard used by `MirrorForegroundService.onCodecModeRequest`
8+
* so it can be unit-tested without spinning up an Android Service. Keeps the
9+
* orchestration (mutex, encoder tear-down, VD swap) in the service while the
10+
* branching logic lives here.
11+
*/
12+
object CodecModeTransition {
13+
14+
const val MODE_H264 = "h264"
15+
const val MODE_MJPEG = "mjpeg"
16+
17+
/**
18+
* @param requestedMode mode string carried by the client control message
19+
* @param currentMode the service's currently active codec mode
20+
* @param jpegEncoderActive whether a JpegEncoder is already live
21+
* @return true if the service should apply the switch (set mode + rebuild)
22+
*/
23+
fun shouldApply(
24+
requestedMode: String,
25+
currentMode: String,
26+
jpegEncoderActive: Boolean
27+
): Boolean {
28+
if (requestedMode != MODE_MJPEG) return false
29+
if (currentMode == MODE_MJPEG && jpegEncoderActive) return false
30+
return true
31+
}
32+
}

‎app/src/main/java/com/castla/mirror/service/MirrorForegroundService.kt‎

Lines changed: 19 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ import com.castla.mirror.utils.LaunchMode
4444
import com.castla.mirror.policy.AutoScaleDecision
4545
import com.castla.mirror.policy.AutoScaleInput
4646
import com.castla.mirror.policy.AutoScalePolicy
47+
import com.castla.mirror.policy.CodecModeTransition
4748
import com.castla.mirror.policy.DisconnectPolicy
4849
import com.castla.mirror.policy.ScreenOffAction
4950
import com.castla.mirror.policy.ScreenOffPolicy
@@ -61,6 +62,8 @@ import kotlinx.coroutines.isActive
6162
import kotlinx.coroutines.flow.MutableStateFlow
6263
import kotlinx.coroutines.flow.StateFlow
6364
import kotlinx.coroutines.launch
65+
import kotlinx.coroutines.sync.Mutex
66+
import kotlinx.coroutines.sync.withLock
6467
import org.json.JSONObject
6568

6669
class MirrorForegroundService : Service() {
@@ -167,7 +170,8 @@ class MirrorForegroundService : Service() {
167170
private var secondaryHeight: Int = 0
168171
private var secondaryRequestedWidth: Int = 0
169172
private var secondaryRequestedHeight: Int = 0
170-
private var currentCodecMode: String = "h264"
173+
@Volatile private var currentCodecMode: String = "h264"
174+
private val pipelineMutex = Mutex()
171175
private var savedMediaVolume: Int = -1
172176
private val mainHandler = Handler(Looper.getMainLooper())
173177
private var splitPresentation: SplitWebPresentation? = null
@@ -2871,7 +2875,7 @@ class MirrorForegroundService : Service() {
28712875
return if (thermalCap != null) minOf(baseMax, thermalCap) else baseMax
28722876
}
28732877

2874-
private suspend fun rebuildPipeline(newWidth: Int, newHeight: Int, force: Boolean = false) {
2878+
private suspend fun rebuildPipeline(newWidth: Int, newHeight: Int, force: Boolean = false) = pipelineMutex.withLock {
28752879
val effectiveMaxHeight = effectiveMaxHeightForRequest(newHeight)
28762880
var cappedWidth = newWidth
28772881
var cappedHeight = newHeight
@@ -2887,12 +2891,12 @@ class MirrorForegroundService : Service() {
28872891

28882892
if (!force && alignedWidth == currentWidth && alignedHeight == currentHeight) {
28892893
Log.d(TAG, "rebuildPipeline skipped: dimensions unchanged ${alignedWidth}x${alignedHeight}")
2890-
return
2894+
return@withLock
28912895
}
28922896

28932897
if (alignedWidth < 320 || alignedWidth > 3840 || alignedHeight < 320 || alignedHeight > 3840) {
28942898
Log.w(TAG, "rebuildPipeline skipped: dimensions out of range ${alignedWidth}x${alignedHeight}")
2895-
return
2899+
return@withLock
28962900
}
28972901

28982902
val width = alignedWidth
@@ -3023,63 +3027,19 @@ class MirrorForegroundService : Service() {
30233027
}
30243028

30253029
private fun onCodecModeRequest(mode: String) {
3026-
if (mode != "mjpeg" || jpegEncoder != null) return
3027-
currentCodecMode = "mjpeg"
3028-
3029-
try {
3030-
val jpeg = JpegEncoder(currentWidth, currentHeight, fps = 15, quality = 65)
3031-
val surface = jpeg.createInputSurface()
3032-
currentEncoderSurface = surface
3033-
3034-
videoEncoder?.release()
3035-
videoEncoder = null
3036-
3037-
jpeg.start { frameData, isKeyFrame -> mirrorServer?.broadcastFrame(frameData, isKeyFrame) }
3038-
jpegEncoder = jpeg
3039-
3040-
if (virtualDisplayManager?.hasVirtualDisplay() == true) {
3041-
dismissSplitPresentation(clearState = false)
3042-
virtualDisplayManager?.releaseVirtualDisplay()
3043-
virtualDisplayManager?.createVirtualDisplay(currentWidth, currentHeight, 160, surface)
3044-
if (virtualDisplayManager?.hasVirtualDisplay() == true) {
3045-
touchInjector?.setVirtualDisplayInjector { action, x, y, pointerId ->
3046-
virtualDisplayManager?.injectInput(action, x, y, pointerId)
3047-
}
3048-
restoreCurrentVdContent()
3049-
} else {
3050-
// Do NOT fall back to MediaProjection here — it would mirror the
3051-
// raw phone screen (showing the Castla UI) instead of virtual
3052-
// display content. Retry VD creation once before giving up.
3053-
Log.w(TAG, "MJPEG VD recreation failed — retrying once")
3054-
virtualDisplayManager?.createVirtualDisplay(currentWidth, currentHeight, 160, surface)
3055-
if (virtualDisplayManager?.hasVirtualDisplay() == true) {
3056-
touchInjector?.setVirtualDisplayInjector { action, x, y, pointerId ->
3057-
virtualDisplayManager?.injectInput(action, x, y, pointerId)
3058-
}
3059-
restoreCurrentVdContent()
3060-
} else {
3061-
Log.e(TAG, "MJPEG VD recreation failed after retry — NOT falling back to MediaProjection")
3062-
}
3063-
}
3064-
} else if (virtualDisplayManager?.isBound() == true) {
3065-
// Shizuku is bound but no VD yet — create one instead of falling back
3066-
virtualDisplayManager?.createVirtualDisplay(currentWidth, currentHeight, 160, surface)
3067-
if (virtualDisplayManager?.hasVirtualDisplay() == true) {
3068-
touchInjector?.setVirtualDisplayInjector { action, x, y, pointerId ->
3069-
virtualDisplayManager?.injectInput(action, x, y, pointerId)
3070-
}
3071-
restoreCurrentVdContent()
3072-
} else {
3073-
Log.e(TAG, "MJPEG VD creation failed — NOT falling back to MediaProjection")
3030+
if (!CodecModeTransition.shouldApply(mode, currentCodecMode, jpegEncoder != null)) return
3031+
currentCodecMode = CodecModeTransition.MODE_MJPEG
3032+
Log.i(TAG, "Codec mode request: mjpeg — delegating to rebuildPipeline")
3033+
serviceScope.launch {
3034+
try {
3035+
rebuildPipeline(currentWidth, currentHeight, force = true)
3036+
if (!singleVdSplit && secondaryWidth > 0 && secondaryHeight > 0) {
3037+
rebuildSecondaryPipeline(secondaryWidth, secondaryHeight)
30743038
}
3075-
} else {
3076-
// Shizuku not available at all — MediaProjection is the only option
3077-
screenCapture?.reconfigure(surface, currentWidth, currentHeight)
3078-
}
3079-
if (!singleVdSplit && secondaryWidth > 0 && secondaryHeight > 0) {
3080-
rebuildSecondaryPipeline(secondaryWidth, secondaryHeight)
3039+
} catch (e: Exception) {
3040+
Log.e(TAG, "Failed to switch codec to mjpeg", e)
30813041
}
3082-
} catch (e: Exception) {}
3042+
}
30833043
}
30843044

30853045
override fun onDestroy() {

‎app/src/test/java/com/castla/mirror/policy/CodecModeTransitionTest.kt‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package com.castla.mirror.policy
2+
3+
import org.junit.Assert.assertFalse
4+
import org.junit.Assert.assertTrue
5+
import org.junit.Test
6+
7+
class CodecModeTransitionTest {
8+
9+
@Test
10+
fun `non-mjpeg request is rejected regardless of state`() {
11+
assertFalse(CodecModeTransition.shouldApply("h264", "h264", jpegEncoderActive = false))
12+
assertFalse(CodecModeTransition.shouldApply("h264", "mjpeg", jpegEncoderActive = true))
13+
assertFalse(CodecModeTransition.shouldApply("", "h264", jpegEncoderActive = false))
14+
assertFalse(CodecModeTransition.shouldApply("av1", "h264", jpegEncoderActive = false))
15+
}
16+
17+
@Test
18+
fun `mjpeg request from h264 applies`() {
19+
assertTrue(CodecModeTransition.shouldApply("mjpeg", "h264", jpegEncoderActive = false))
20+
}
21+
22+
@Test
23+
fun `mjpeg request when already mjpeg with active encoder is a no-op`() {
24+
assertFalse(CodecModeTransition.shouldApply("mjpeg", "mjpeg", jpegEncoderActive = true))
25+
}
26+
27+
@Test
28+
fun `mjpeg request when mode is mjpeg but encoder is missing still applies`() {
29+
// Covers the case where the mode flag was set but the previous rebuild
30+
// failed to finish creating the JpegEncoder — the next request must
31+
// still trigger a rebuild instead of silently skipping.
32+
assertTrue(CodecModeTransition.shouldApply("mjpeg", "mjpeg", jpegEncoderActive = false))
33+
}
34+
}

0 commit comments

Comments
 (0)