Skip to content

Commit 13744a3

Browse files
jcgglclaude
jcggl
and
claude
committed
feat: case-insensitive ARKit mapping + reInferWithEmotion for file playback
- Add resolvedArkit52 mapping table: supports both camelCase (jawOpen) and PascalCase (JawOpen) ARKit blendshape names from third-party VRM models - Add debounced reInferWithEmotion() to file playback mode in guide and vanilla-avatar pages (200ms debounce + _reInferring guard) - Fix vanilla-comparison resetVRM() to use resolvedArkit52 instead of ARKIT_52 - Sync guide source code viewer with runtime emotion/blendshape logic Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 3269d13 commit 13744a3

File tree

3 files changed

+86
-24
lines changed

3 files changed

+86
-24
lines changed

‎examples/guide/index.html‎

Lines changed: 56 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -995,6 +995,7 @@ <h2 class="step-title">Add Real-time Microphone</h2>
995995
let fileResult = null, fileStartTime = 0, filePlaying = false; // time-synced file playback
996996
let micActive = false, micAudioCtx = null, micWorklet = null, micStream = null;
997997
let useVrmMode = false;
998+
let resolvedArkit52 = [...ARKIT_52]; // mapped to model's actual expression names
998999
let idleGenerator = null, idleClock = 0;
9991000
let prevFrame = null;
10001001

@@ -1083,14 +1084,26 @@ <h2 class="step-title">Add Real-time Microphone</h2>
10831084
}
10841085

10851086
// Auto-detect VRM expression mode (ARKit 52-dim vs VRM 18-dim)
1087+
// Case-insensitive matching: supports camelCase (jawOpen) and PascalCase (JawOpen) ARKit names
10861088
useVrmMode = false;
1089+
resolvedArkit52 = [...ARKIT_52];
10871090
if (vrm.expressionManager) {
10881091
const exprNames = Object.keys(vrm.expressionManager.expressionMap || {});
1089-
const arkitNames = ['jawOpen','mouthFunnel','mouthPucker','eyeBlinkLeft','eyeBlinkRight'];
1090-
const hasArkit = arkitNames.filter(n => exprNames.includes(n)).length >= 3;
1092+
const exprLower = {};
1093+
for (const n of exprNames) exprLower[n.toLowerCase()] = n;
1094+
1095+
const arkitProbe = ['jawOpen','mouthFunnel','mouthPucker','eyeBlinkLeft','eyeBlinkRight'];
1096+
const hasArkit = arkitProbe.filter(n => exprLower[n.toLowerCase()]).length >= 3;
10911097
const vrmPresetNames = ['aa','ih','ou','ee','oh','happy','angry','sad','relaxed','surprised'];
10921098
const hasVrmPreset = vrmPresetNames.filter(n => exprNames.includes(n)).length >= 3;
1093-
if (!hasArkit && hasVrmPreset) useVrmMode = true;
1099+
1100+
if (hasArkit) {
1101+
// Build resolved name array: map each ARKit name to model's actual casing
1102+
resolvedArkit52 = ARKIT_52.map(n => exprLower[n.toLowerCase()] || n);
1103+
useVrmMode = false;
1104+
} else if (hasVrmPreset) {
1105+
useVrmMode = true;
1106+
}
10941107
console.log(`[VRM] Mode: ${useVrmMode ? 'VRM 18-dim' : 'ARKit 52-dim'} (arkit=${hasArkit}, vrm=${hasVrmPreset})`);
10951108
}
10961109

@@ -1207,11 +1220,25 @@ <h2 class="step-title">Add Real-time Microphone</h2>
12071220
const emotionSliders = EMOTION_KEYS.map(k => $(`emo-${k}`));
12081221
const emotionVals = EMOTION_KEYS.map(k => $(`emo-${k}-val`));
12091222

1223+
let _reInferTimer = null;
1224+
let _reInferring = false;
12101225
function updateEmotionVector() {
12111226
const vec = emotionSliders.map(s => parseInt(s.value) / 100);
12121227
if (!lipsync?.setEmotion) return;
12131228
try { lipsync.setEmotion(vec); } catch (e) { console.warn('setEmotion:', e.message); }
12141229
if (micActive) frameQueue.length = 0;
1230+
// File playback: debounced re-infer (prevents rapid-fire during slider drag)
1231+
if (filePlaying && fileResult && lipsync.reInferWithEmotion && !_reInferring) {
1232+
clearTimeout(_reInferTimer);
1233+
_reInferTimer = setTimeout(() => {
1234+
const v = emotionSliders.map(s => parseInt(s.value) / 100);
1235+
_reInferring = true;
1236+
lipsync.reInferWithEmotion(v)
1237+
.then(r => { if (filePlaying) fileResult = r; })
1238+
.catch(() => {})
1239+
.finally(() => { _reInferring = false; });
1240+
}, 200);
1241+
}
12151242
}
12161243

12171244
function showEmotionPanel(show) {
@@ -1396,7 +1423,7 @@ <h2 class="step-title">Add Real-time Microphone</h2>
13961423
// ════════════════════════════════════════
13971424
function applyArkitBlendshapes(frame) {
13981425
if (!vrm?.expressionManager) return;
1399-
for (let i = 0; i < Math.min(frame.length, ARKIT_52.length); i++) vrm.expressionManager.setValue(ARKIT_52[i], frame[i]);
1426+
for (let i = 0; i < Math.min(frame.length, resolvedArkit52.length); i++) vrm.expressionManager.setValue(resolvedArkit52[i], frame[i]);
14001427
}
14011428
function applyVrmBlendshapes(vrmFrame) {
14021429
if (!vrm?.expressionManager || !vrmFrame) return;
@@ -1411,7 +1438,7 @@ <h2 class="step-title">Add Real-time Microphone</h2>
14111438
}
14121439
function resetVRM() {
14131440
if (!vrm?.expressionManager) return;
1414-
const names = useVrmMode ? VRM_EXPRESSIONS : ARKIT_52;
1441+
const names = useVrmMode ? VRM_EXPRESSIONS : resolvedArkit52;
14151442
for (const n of names) vrm.expressionManager.setValue(n, 0);
14161443
prevFrame = null;
14171444
}
@@ -1688,6 +1715,7 @@ <h2 class="step-title">Add Real-time Microphone</h2>
16881715
'let fileResult = null, fileStartTime = 0, filePlaying = false;',
16891716
'let micActive = false, micAudioCtx = null, micWorklet = null, micStream = null;',
16901717
'let useVrmMode = false;',
1718+
'let resolvedArkit52 = [...ARKIT_52];',
16911719
'let idleGenerator = null, idleClock = 0;',
16921720
'let prevFrame = null;',
16931721
'',
@@ -1773,15 +1801,17 @@ <h2 class="step-title">Add Real-time Microphone</h2>
17731801
" } catch (e) { console.warn('VRMA skip:', e.message); }",
17741802
' }',
17751803
'',
1776-
' // Auto-detect VRM expression mode (ARKit 52-dim vs VRM 18-dim)',
1777-
' useVrmMode = false;',
1804+
' // Auto-detect VRM expression mode (case-insensitive ARKit 52 vs VRM 18-dim)',
1805+
' useVrmMode = false; resolvedArkit52 = [...ARKIT_52];',
17781806
' if (vrm.expressionManager) {',
17791807
' const exprNames = Object.keys(vrm.expressionManager.expressionMap || {});',
1780-
" const arkitNames = ['jawOpen','mouthFunnel','mouthPucker','eyeBlinkLeft','eyeBlinkRight'];",
1781-
' const hasArkit = arkitNames.filter(n => exprNames.includes(n)).length >= 3;',
1808+
' const exprLower = {}; for (const n of exprNames) exprLower[n.toLowerCase()] = n;',
1809+
" const arkitProbe = ['jawOpen','mouthFunnel','mouthPucker','eyeBlinkLeft','eyeBlinkRight'];",
1810+
' const hasArkit = arkitProbe.filter(n => exprLower[n.toLowerCase()]).length >= 3;',
17821811
" const vrmNames = ['aa','ih','ou','ee','oh','happy','angry','sad','relaxed','surprised'];",
17831812
' const hasVrm = vrmNames.filter(n => exprNames.includes(n)).length >= 3;',
1784-
' if (!hasArkit && hasVrm) useVrmMode = true;',
1813+
' if (hasArkit) { resolvedArkit52 = ARKIT_52.map(n => exprLower[n.toLowerCase()] || n); }',
1814+
' else if (hasVrm) useVrmMode = true;',
17851815
' }',
17861816
'',
17871817
" vrmBadge.textContent = 'VRM loaded (' + (useVrmMode ? 'VRM' : 'ARKit') + ')';",
@@ -1943,21 +1973,29 @@ <h2 class="step-title">Add Real-time Microphone</h2>
19431973
'}',
19441974
'',
19451975
'// \u2500\u2500 Emotion Control (V2 only) \u2500\u2500',
1946-
'// setEmotion([neutral, joy, anger, sadness, surprise]) — values 0-1',
1947-
'// Mic streaming: emotion applies per-chunk in real time',
1948-
'// File playback: set emotion before uploading audio',
1976+
'// Mic: setEmotion() + queue flush | File: debounced reInferWithEmotion()',
1977+
'let _reInferTimer = null, _reInferring = false;',
19491978
'function setEmotionFromUI(vec) {',
19501979
' if (!lipsync?.setEmotion) return;',
19511980
' try { lipsync.setEmotion(vec); } catch (e) { console.warn(e); }',
1952-
' if (micActive) frameQueue.length = 0; // flush old-emotion frames',
1981+
' if (micActive) frameQueue.length = 0;',
1982+
' if (filePlaying && fileResult && lipsync.reInferWithEmotion && !_reInferring) {',
1983+
' clearTimeout(_reInferTimer);',
1984+
' _reInferTimer = setTimeout(() => {',
1985+
' _reInferring = true;',
1986+
' lipsync.reInferWithEmotion(vec)',
1987+
' .then(r => { if (filePlaying) fileResult = r; })',
1988+
' .catch(() => {})',
1989+
' .finally(() => { _reInferring = false; });',
1990+
' }, 200);',
1991+
' }',
19531992
'}',
1954-
'// Example: setEmotionFromUI([0, 0.8, 0, 0, 0]); // 80% joy',
19551993
'',
19561994
'// \u2500\u2500 Blendshape Helpers \u2500\u2500',
19571995
'function applyArkitBlendshapes(frame) {',
19581996
' if (!vrm?.expressionManager) return;',
1959-
' for (let i = 0; i < Math.min(frame.length, ARKIT_52.length); i++)',
1960-
' vrm.expressionManager.setValue(ARKIT_52[i], frame[i]);',
1997+
' for (let i = 0; i < Math.min(frame.length, resolvedArkit52.length); i++)',
1998+
' vrm.expressionManager.setValue(resolvedArkit52[i], frame[i]);',
19611999
'}',
19622000
'function applyVrmBlendshapes(vrmFrame) {',
19632001
' if (!vrm?.expressionManager || !vrmFrame) return;',
@@ -1971,7 +2009,7 @@ <h2 class="step-title">Add Real-time Microphone</h2>
19712009
'}',
19722010
'function resetVRM() {',
19732011
' if (!vrm?.expressionManager) return;',
1974-
' const names = useVrmMode ? VRM_EXPRESSIONS : ARKIT_52;',
2012+
' const names = useVrmMode ? VRM_EXPRESSIONS : resolvedArkit52;',
19752013
' for (const n of names) vrm.expressionManager.setValue(n, 0);',
19762014
' prevFrame = null;',
19772015
'}',

‎examples/vanilla-avatar/index.html‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -456,10 +456,23 @@ <h2>52 ARKit Blendshapes — V2 Emotion</h2>
456456
const emoVals = document.querySelectorAll('.emo-val');
457457
const emoPresetBtns = document.querySelectorAll('.emo-presets button');
458458

459+
let _reInferTimer = null, _reInferring = false;
459460
function applyEmotion() {
460461
if (!lipsync?.setEmotion) return;
461462
const vec = Array.from(emoSliders).map(s => parseInt(s.value) / 100);
462463
try { lipsync.setEmotion(vec); } catch (e) { console.warn('setEmotion:', e); }
464+
// Re-infer cached audio with new emotion (debounced, non-blocking)
465+
if (currentResult && lipsync.reInferWithEmotion && !_reInferring) {
466+
clearTimeout(_reInferTimer);
467+
_reInferTimer = setTimeout(() => {
468+
const v = Array.from(emoSliders).map(s => parseInt(s.value) / 100);
469+
_reInferring = true;
470+
lipsync.reInferWithEmotion(v)
471+
.then(r => { currentResult = r; })
472+
.catch(() => {})
473+
.finally(() => { _reInferring = false; });
474+
}, 200);
475+
}
463476
}
464477

465478
emoSliders.forEach((s, i) => s.addEventListener('input', () => {

‎examples/vanilla-comparison/index.html‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -393,6 +393,7 @@ <h1>Anima<span>Sync</span></h1>
393393
let isSpeaking = false;
394394
let crossFadeProgress = 0;
395395
let useVrmMode = false;
396+
let resolvedArkit52 = [...ARKIT_52]; // mapped to model's actual expression names
396397

397398
const VRM_EXPRESSIONS = ['aa','ih','ou','ee','oh','happy','angry','sad','relaxed','surprised','neutral','blinkLeft','blinkRight','lookUp','lookDown','lookLeft','lookRight','surprised'];
398399

@@ -426,13 +427,23 @@ <h1>Anima<span>Sync</span></h1>
426427
} catch (e) { console.warn('VRMA skip:', e.message); }
427428

428429
// Auto-detect VRM expression mode (first load only)
430+
// Case-insensitive matching: supports camelCase (jawOpen) and PascalCase (JawOpen) ARKit names
429431
if (!useVrmMode && vrm.expressionManager) {
430432
const exprNames = Object.keys(vrm.expressionManager.expressionMap || {});
431-
const arkitNames = ['jawOpen','mouthFunnel','mouthPucker','eyeBlinkLeft','eyeBlinkRight'];
432-
const hasArkit = arkitNames.filter(n => exprNames.includes(n)).length >= 3;
433+
const exprLower = {};
434+
for (const n of exprNames) exprLower[n.toLowerCase()] = n;
435+
436+
const arkitProbe = ['jawOpen','mouthFunnel','mouthPucker','eyeBlinkLeft','eyeBlinkRight'];
437+
const hasArkit = arkitProbe.filter(n => exprLower[n.toLowerCase()]).length >= 3;
433438
const vrmPresetNames = ['aa','ih','ou','ee','oh','happy','angry','sad','relaxed','surprised'];
434439
const hasVrmPreset = vrmPresetNames.filter(n => exprNames.includes(n)).length >= 3;
435-
if (!hasArkit && hasVrmPreset) useVrmMode = true;
440+
441+
if (hasArkit) {
442+
resolvedArkit52 = ARKIT_52.map(n => exprLower[n.toLowerCase()] || n);
443+
useVrmMode = false;
444+
} else if (hasVrmPreset) {
445+
useVrmMode = true;
446+
}
436447
console.log(`[VRM] Mode: ${useVrmMode ? 'VRM 18-dim' : 'ARKit 52-dim'} (arkit=${hasArkit}, vrm=${hasVrmPreset})`);
437448
}
438449

@@ -458,7 +469,7 @@ <h1>Anima<span>Sync</span></h1>
458469

459470
function resetVRM(vrm) {
460471
if (!vrm?.expressionManager) return;
461-
const names = useVrmMode ? VRM_EXPRESSIONS : ARKIT_52;
472+
const names = useVrmMode ? VRM_EXPRESSIONS : resolvedArkit52;
462473
for (const name of names) vrm.expressionManager.setValue(name, 0);
463474
}
464475

@@ -697,8 +708,8 @@ <h1>Anima<span>Sync</span></h1>
697708
vrm.expressionManager.setValue(VRM_EXPRESSIONS[i], frameData.vrm[i] || 0);
698709
}
699710
} else {
700-
for (let i = 0; i < Math.min(frameData.arkit.length, ARKIT_52.length); i++) {
701-
vrm.expressionManager.setValue(ARKIT_52[i], frameData.arkit[i]);
711+
for (let i = 0; i < Math.min(frameData.arkit.length, resolvedArkit52.length); i++) {
712+
vrm.expressionManager.setValue(resolvedArkit52[i], frameData.arkit[i]);
702713
}
703714
}
704715
}

0 commit comments

Comments
 (0)