Skip to content

Commit b110434

Browse files
joewinkeclaude
joewinke
and
claude
committed
task(jat-1f6dt): Add Voice Inbox — ollama organizes transcripts into suggested tasks before creation
Instead of auto-creating a task immediately after transcription, the voice API now calls ollama (qwen2.5:7b) to parse the transcript into structured SuggestedTask[] JSON and appends them to a JSONL timeline file. VoiceInbox.svelte mounts EventStack on that virtual session and lets the user review and bulk-create tasks via /api/tasks/bulk. Falls back to direct task creation when ollama is unavailable. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 73f59df commit b110434

File tree

3 files changed

+233
-73
lines changed

3 files changed

+233
-73
lines changed

‎ide/src/lib/components/voice/VoiceInbox.svelte‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
<script lang="ts">
2+
/**
3+
* VoiceInbox
4+
*
5+
* Shows pending voice suggestions from the voice inbox timeline.
6+
* The voice API transcribes audio, runs organize-tasks via ollama,
7+
* and writes SuggestedTask[] events to /tmp/jat-timeline-jat-voice.jsonl.
8+
*
9+
* This component mounts EventStack on that virtual session so the user
10+
* can review, edit, and selectively import the suggested tasks.
11+
*/
12+
13+
import EventStack from '$lib/components/work/EventStack.svelte';
14+
import type { SuggestedTaskWithState } from '$lib/types/signals';
15+
import { successToast, errorToast } from '$lib/stores/toasts.svelte';
16+
17+
let { availableProjects = [], defaultProject = '' }: {
18+
availableProjects?: string[];
19+
defaultProject?: string;
20+
} = $props();
21+
22+
async function createSuggestedTasks(
23+
tasks: SuggestedTaskWithState[]
24+
): Promise<{ success: any[]; failed: any[] }> {
25+
if (tasks.length === 0) return { success: [], failed: [] };
26+
27+
const tasksToCreate = tasks.map((t) => ({
28+
type: t.edits?.type || t.type || 'task',
29+
title: t.edits?.title || t.title,
30+
description: t.edits?.description || t.description || '',
31+
priority: t.edits?.priority ?? t.priority ?? 2,
32+
project: t.edits?.project || t.project || defaultProject || undefined,
33+
labels: t.edits?.labels || t.labels || undefined,
34+
depends_on: t.edits?.depends_on || t.depends_on || undefined
35+
}));
36+
37+
try {
38+
const response = await fetch('/api/tasks/bulk', {
39+
method: 'POST',
40+
headers: { 'Content-Type': 'application/json' },
41+
body: JSON.stringify({ tasks: tasksToCreate })
42+
});
43+
44+
if (!response.ok) {
45+
const err = await response.json();
46+
throw new Error(err.message || 'Failed to create tasks');
47+
}
48+
49+
const data = await response.json();
50+
const results = {
51+
success: data.results?.filter((r: any) => r.success) || [],
52+
failed: data.results?.filter((r: any) => !r.success) || []
53+
};
54+
55+
if (results.success.length > 0) {
56+
successToast(`Created ${results.success.length} task${results.success.length > 1 ? 's' : ''} from voice note`);
57+
}
58+
if (results.failed.length > 0) {
59+
errorToast(`Failed to create ${results.failed.length} task(s)`);
60+
}
61+
62+
return results;
63+
} catch (err: any) {
64+
errorToast(err.message);
65+
return {
66+
success: [],
67+
failed: tasks.map((t) => ({ title: t.title, error: err.message }))
68+
};
69+
}
70+
}
71+
</script>
72+
73+
<!-- EventStack renders nothing when there are no voice events -->
74+
<EventStack
75+
sessionName="jat-voice"
76+
layoutMode="inline"
77+
pollInterval={15000}
78+
onCreateTasks={createSuggestedTasks}
79+
{availableProjects}
80+
{defaultProject}
81+
/>

‎ide/src/routes/api/tasks/voice/+server.js‎

Lines changed: 145 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import { createTask } from '$lib/server/jat-tasks.js';
1515
import { invalidateCache } from '$lib/server/cache.js';
1616
import { _resetTaskCache } from '../../../api/agents/+server.js';
1717
import { emitEvent } from '$lib/utils/eventBus.server.js';
18-
import { writeFileSync, unlinkSync, mkdirSync, statSync } from 'fs';
18+
import { writeFileSync, unlinkSync, mkdirSync, statSync, appendFileSync } from 'fs';
1919
import { exec, execSync } from 'child_process';
2020
import { randomBytes } from 'crypto';
2121
import { join } from 'path';
@@ -49,13 +49,91 @@ function getAudioDate(filePath) {
4949
return new Date();
5050
}
5151

52+
const VOICE_TIMELINE_FILE = '/tmp/jat-timeline-jat-voice.jsonl';
53+
54+
/**
55+
* Call local ollama to organize a transcript into structured tasks (JSON).
56+
* Returns an array of SuggestedTask objects.
57+
* @param {string} transcript
58+
* @returns {Promise<Array>}
59+
*/
60+
async function organizeTranscript(transcript) {
61+
const prompt = `You are a task organizer. Extract every actionable task from this voice note transcript.
62+
63+
Return ONLY a JSON object with this exact structure (no markdown, no explanation):
64+
{
65+
"tasks": [
66+
{
67+
"type": "task",
68+
"title": "Short actionable title in imperative form",
69+
"description": "More context if the transcript provides it",
70+
"priority": 2,
71+
"labels": "voice"
72+
}
73+
]
74+
}
75+
76+
Types: task, feature, bug, chore
77+
Priority: 0=critical 1=high 2=medium 3=low 4=lowest
78+
79+
Use priority cues from the transcript ("urgent", "first thing", "must", "deadline" → lower number).
80+
Group related items into one task rather than splitting trivially.
81+
82+
Transcript:
83+
${transcript}`;
84+
85+
const response = await fetch('http://localhost:11434/api/generate', {
86+
method: 'POST',
87+
headers: { 'Content-Type': 'application/json' },
88+
body: JSON.stringify({
89+
model: process.env.ORGANIZE_TASKS_MODEL || 'qwen2.5:7b',
90+
prompt,
91+
format: 'json',
92+
stream: false,
93+
options: { temperature: 0.3, num_predict: 2048 }
94+
}),
95+
signal: AbortSignal.timeout(120_000)
96+
});
97+
98+
if (!response.ok) {
99+
throw new Error(`ollama request failed: ${response.status}`);
100+
}
101+
102+
const data = await response.json();
103+
const parsed = JSON.parse(data.response);
104+
const tasks = parsed.tasks;
105+
106+
if (!Array.isArray(tasks) || tasks.length === 0) {
107+
throw new Error('ollama returned no tasks');
108+
}
109+
110+
return tasks;
111+
}
112+
113+
/**
114+
* Append organized tasks to the voice inbox timeline file.
115+
* EventStack polls /api/sessions/jat-voice/timeline which reads this file.
116+
* @param {Array} tasks
117+
*/
118+
function appendToVoiceTimeline(tasks) {
119+
mkdirSync(TEMP_DIR, { recursive: true });
120+
const event = {
121+
type: 'tasks',
122+
session_id: 'voice',
123+
tmux_session: 'jat-voice',
124+
timestamp: new Date().toISOString(),
125+
data: tasks
126+
};
127+
appendFileSync(VOICE_TIMELINE_FILE, JSON.stringify(event) + '\n');
128+
}
129+
52130
/**
53-
* Transcribe audio file and create task (runs in background).
131+
* Transcribe audio file and organize into suggested tasks (runs in background).
54132
* @param {string} audioPath
55133
* @param {string} title
56134
* @param {number} priority
57135
*/
58-
function transcribeAndCreateTask(audioPath, title, priority) {
136+
function transcribeAndOrganize(audioPath, title, priority) {
59137
const id = randomBytes(4).toString('hex');
60138
const wavPath = join(TEMP_DIR, `transcribe-${id}.wav`);
61139

@@ -77,7 +155,7 @@ function transcribeAndCreateTask(audioPath, title, priority) {
77155
timeout: 600_000,
78156
encoding: 'utf-8',
79157
maxBuffer: 10 * 1024 * 1024
80-
}, (transcribeErr, stdout) => {
158+
}, async (transcribeErr, stdout) => {
81159
// Clean up wav
82160
try { unlinkSync(wavPath); } catch {}
83161

@@ -98,41 +176,40 @@ function transcribeAndCreateTask(audioPath, title, priority) {
98176
return;
99177
}
100178

101-
// Step 3: Create the task
102-
const projectPath = process.cwd().replace(/\/ide$/, '');
103-
179+
// Step 3: Organize transcript into structured tasks via ollama
104180
try {
105-
const createdTask = createTask({
106-
projectPath,
107-
title,
108-
description: text,
109-
type: 'task',
110-
priority: isNaN(priority) ? 2 : Math.max(0, Math.min(4, priority)),
111-
labels: ['voice'],
112-
deps: [],
113-
assignee: null,
114-
notes: ''
115-
});
116-
117-
invalidateCache.tasks();
118-
invalidateCache.agents();
119-
_resetTaskCache();
120-
121-
emitEvent({
122-
type: 'task_created',
123-
source: 'voice_api',
124-
data: {
125-
taskId: createdTask.id,
181+
const tasks = await organizeTranscript(text);
182+
appendToVoiceTimeline(tasks);
183+
console.log(`[voice] Organized ${tasks.length} task(s) into voice inbox`);
184+
} catch (organizeErr) {
185+
console.error('[voice] organize failed, falling back to single task:', organizeErr.message);
186+
187+
// Fallback: create a single task from the transcript directly
188+
const projectPath = process.cwd().replace(/\/ide$/, '');
189+
try {
190+
const createdTask = createTask({
191+
projectPath,
126192
title,
193+
description: text,
127194
type: 'task',
128-
priority,
129-
labels: ['voice']
130-
}
131-
});
132-
133-
console.log(`[voice] Task ${createdTask.id} created: "${title}"`);
134-
} catch (e) {
135-
console.error('[voice] Failed to create task:', e);
195+
priority: isNaN(priority) ? 2 : Math.max(0, Math.min(4, priority)),
196+
labels: ['voice'],
197+
deps: [],
198+
assignee: null,
199+
notes: ''
200+
});
201+
invalidateCache.tasks();
202+
invalidateCache.agents();
203+
_resetTaskCache();
204+
emitEvent({
205+
type: 'task_created',
206+
source: 'voice_api',
207+
data: { taskId: createdTask.id, title, type: 'task', priority, labels: ['voice'] }
208+
});
209+
console.log(`[voice] Fallback: task ${createdTask.id} created: "${title}"`);
210+
} catch (e) {
211+
console.error('[voice] Fallback task creation also failed:', e);
212+
}
136213
}
137214
});
138215
});
@@ -144,52 +221,47 @@ export async function POST({ request }) {
144221

145222
try {
146223
if (contentType.includes('application/json')) {
147-
// JSON body — synchronous, create task immediately
224+
// JSON body — pre-transcribed text, organize async and add to voice inbox
148225
const body = await request.json();
149226
const text = body.text?.trim();
150227

151228
if (!text) {
152229
return json({ error: true, message: 'Missing "text" field' }, { status: 400 });
153230
}
154231

155-
const now = new Date();
156-
const timestamp = now.toLocaleString('en-US', {
157-
month: 'short', day: 'numeric', year: 'numeric',
158-
hour: 'numeric', minute: '2-digit', hour12: true
159-
});
160-
const title = body.title?.trim() || `Voice note ${timestamp}`;
161-
const priority = body.priority !== undefined ? parseInt(body.priority) : 2;
162-
const projectPath = process.cwd().replace(/\/ide$/, '');
163-
164-
const createdTask = createTask({
165-
projectPath,
166-
title,
167-
description: text,
168-
type: 'task',
169-
priority: isNaN(priority) ? 2 : Math.max(0, Math.min(4, priority)),
170-
labels: ['voice'],
171-
deps: [],
172-
assignee: null,
173-
notes: ''
174-
});
175-
176-
invalidateCache.tasks();
177-
invalidateCache.agents();
178-
_resetTaskCache();
179-
180-
try {
181-
emitEvent({
182-
type: 'task_created',
183-
source: 'voice_api',
184-
data: { taskId: createdTask.id, title, type: 'task', priority, labels: ['voice'] }
232+
// Organize in background, don't block the response
233+
organizeTranscript(text).then((tasks) => {
234+
appendToVoiceTimeline(tasks);
235+
console.log(`[voice] Organized ${tasks.length} task(s) from text into voice inbox`);
236+
}).catch((err) => {
237+
console.error('[voice] organize failed for text input:', err.message);
238+
// Fallback: single task
239+
const now = new Date();
240+
const timestamp = now.toLocaleString('en-US', {
241+
month: 'short', day: 'numeric', year: 'numeric',
242+
hour: 'numeric', minute: '2-digit', hour12: true
185243
});
186-
} catch {}
244+
const title = body.title?.trim() || `Voice note ${timestamp}`;
245+
const priority = body.priority !== undefined ? parseInt(body.priority) : 2;
246+
const projectPath = process.cwd().replace(/\/ide$/, '');
247+
try {
248+
const createdTask = createTask({
249+
projectPath, title, description: text, type: 'task',
250+
priority: isNaN(priority) ? 2 : Math.max(0, Math.min(4, priority)),
251+
labels: ['voice'], deps: [], assignee: null, notes: ''
252+
});
253+
invalidateCache.tasks();
254+
invalidateCache.agents();
255+
_resetTaskCache();
256+
} catch (e) {
257+
console.error('[voice] Fallback task creation failed:', e);
258+
}
259+
});
187260

188261
return json({
189262
success: true,
190-
task: createdTask,
191-
message: `Task ${createdTask.id} created from voice note`
192-
}, { status: 201 });
263+
message: 'Voice note received — organizing in background. Tasks will appear in Voice Inbox shortly.'
264+
}, { status: 202 });
193265

194266
} else {
195267
// Audio file — save and process async
@@ -240,8 +312,8 @@ export async function POST({ request }) {
240312
});
241313
if (!title) title = `Voice note ${timestamp}`;
242314

243-
// Fire and forget — transcription happens in background
244-
transcribeAndCreateTask(audioTempPath, title, priority);
315+
// Fire and forget — transcription + organize happens in background
316+
transcribeAndOrganize(audioTempPath, title, priority);
245317

246318
return json({
247319
success: true,

‎ide/src/routes/tasks/+page.svelte‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
groupTasksByDay,
3434
} from "$lib/utils/completedTaskHelpers";
3535
import { isHumanTask } from "$lib/utils/badgeHelpers";
36+
import VoiceInbox from "$lib/components/voice/VoiceInbox.svelte";
3637
3738
interface TmuxSession {
3839
name: string;
@@ -1671,6 +1672,12 @@
16711672
<span>No projects with active sessions or open tasks</span>
16721673
</div>
16731674
{:else}
1675+
<!-- Voice Inbox: pending suggestions from iOS voice notes (hidden when empty) -->
1676+
<VoiceInbox
1677+
availableProjects={projects}
1678+
defaultProject={selectedProject || ''}
1679+
/>
1680+
16741681
<!-- Selected Project Content -->
16751682
{#if selectedProject}
16761683
{@const projectSessions =

0 commit comments

Comments
 (0)