Skip to content

Commit 99a9070

Browse files
jan-beranekjan-beranek
committed
feat: add wopee-dino support
1 parent dd3dbf1 commit 99a9070

File tree

1 file changed

+58
-41
lines changed

1 file changed

+58
-41
lines changed

‎app/src/core/api-client.js‎

Lines changed: 58 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -79,46 +79,51 @@ export class ApiClient {
7979
let res;
8080
attemptKind = 'single';
8181
if (endpointType === 'groundingdino') {
82-
// For GroundingDINO servers that expect multipart/form-data
83-
// Send exactly the fields the reference curl uses: file, prompt, thresholds
84-
const fd = new FormData();
85-
const mime = imageBlob?.type || 'image/png';
86-
const ext = mime.includes('jpeg') ? 'jpg' : (mime.split('/')[1] || 'png');
87-
const fname = `image.${ext}`;
88-
const p = String(prompt ?? '');
89-
const filePart = (imageBlob instanceof File) ? imageBlob : new File([imageBlob], fname, { type: mime });
90-
fd.append('file', filePart, fname);
91-
fd.append('prompt', p);
92-
if (dinoBoxThreshold != null) fd.append('box_threshold', String(dinoBoxThreshold));
93-
if (dinoTextThreshold != null) fd.append('text_threshold', String(dinoTextThreshold));
94-
// Remove JSON content-type so browser sets multipart boundary
95-
headers = this._sanitizeForMultipart(headers);
96-
res = await fetch(url, { method: 'POST', headers, body: fd, signal: controller.signal });
97-
// If server rejects multipart with generic client/server errors (not validation for file/prompt), retry JSON
98-
let contentType0 = res.headers.get('content-type') || '';
99-
let j0 = null;
100-
if (contentType0.includes('application/json')) {
101-
try { j0 = await res.clone().json(); } catch {}
102-
} else {
82+
// For GroundingDINO servers that expect multipart/form-data.
83+
// Prefer the "image" field (as in newer servers), then fall back to "file".
84+
const buildForm = (fieldName) => {
85+
const fd = new FormData();
86+
const mime = imageBlob?.type || 'image/png';
87+
const ext = mime.includes('jpeg') ? 'jpg' : (mime.split('/')[1] || 'png');
88+
const fname = `image.${ext}`;
89+
const p = String(prompt ?? '');
90+
const filePart = (imageBlob instanceof File) ? imageBlob : new File([imageBlob], fname, { type: mime });
91+
fd.append(fieldName, filePart, fname);
92+
fd.append('prompt', p);
93+
if (dinoBoxThreshold != null) fd.append('box_threshold', String(dinoBoxThreshold));
94+
if (dinoTextThreshold != null) fd.append('text_threshold', String(dinoTextThreshold));
95+
return fd;
96+
};
97+
98+
headers = this._sanitizeForMultipart(headers); // remove JSON content-type so browser sets multipart boundary
99+
// Attempt 1: send as 'image'
100+
res = await fetch(url, { method: 'POST', headers, body: buildForm('image'), signal: controller.signal });
101+
let attempt = 'multipart-image';
102+
103+
// If server rejects multipart with generic client/server errors, try 'file' field next
104+
if (!res.ok && [400, 401, 403, 404, 405, 406, 415, 422].includes(res.status)) {
103105
try { await res.clone().text(); } catch {}
104-
}
105-
const missingFileOrPrompt = !!(j0 && Array.isArray(j0.detail) && j0.detail.some(d => {
106-
const loc = String(d?.loc?.join('.'));
107-
return loc.includes('file') || loc.includes('prompt');
108-
}));
109-
const shouldRetryJson = (!res.ok && [400, 401, 403, 404, 405, 406, 415].includes(res.status));
110-
if (shouldRetryJson) {
111-
const jsonBody = buildRequestBody({ endpointType, baseURL, model, temperature, maxTokens, prompt, sysPrompt, imageB64: b64, reasoningEffort, dinoBoxThreshold, dinoTextThreshold });
112-
const jsonHeaders = { 'Content-Type': 'application/json' };
113106
const controller2 = new AbortController();
114107
const to2 = setTimeout(() => controller2.abort('timeout'), timeoutMs);
115-
const res2 = await fetch(url, { method: 'POST', headers: jsonHeaders, body: JSON.stringify(jsonBody), signal: controller2.signal });
108+
const res2 = await fetch(url, { method: 'POST', headers, body: buildForm('file'), signal: controller2.signal });
116109
clearTimeout(to2);
117110
res = res2;
118-
attemptKind = 'retry-json';
119-
} else if (!res.ok && res.status === 422 && missingFileOrPrompt) {
120-
// Keep the original error; do not switch to JSON because server expects multipart
111+
attempt = 'multipart-file';
112+
}
113+
114+
// If still not OK and clearly a client/server issue, retry with JSON body
115+
if (!res.ok && [400, 401, 403, 404, 405, 406, 415].includes(res.status)) {
116+
const jsonBody = buildRequestBody({ endpointType, baseURL, model, temperature, maxTokens, prompt, sysPrompt, imageB64: b64, reasoningEffort, dinoBoxThreshold, dinoTextThreshold });
117+
const jsonHeaders = { 'Content-Type': 'application/json' };
118+
const controller3 = new AbortController();
119+
const to3 = setTimeout(() => controller3.abort('timeout'), timeoutMs);
120+
const res3 = await fetch(url, { method: 'POST', headers: jsonHeaders, body: JSON.stringify(jsonBody), signal: controller3.signal });
121+
clearTimeout(to3);
122+
res = res3;
123+
attempt = `${attempt} -> retry-json`;
121124
}
125+
126+
attemptKind = attempt;
122127
} else {
123128
res = await fetch(url, { method: 'POST', headers, body: JSON.stringify(body), signal: controller.signal });
124129
}
@@ -195,7 +200,9 @@ export class ApiClient {
195200
url,
196201
headers: this._sanitizeHeaders(headers),
197202
bodyPreview: (endpointType === 'groundingdino')
198-
? (attemptKind === 'retry-json' ? 'multipart (initial) -> retried JSON (image+prompt+thresholds)' : 'multipart/form-data (file, prompt, thresholds)')
203+
? (attemptKind.includes('retry-json')
204+
? `${attemptKind} (prompt, thresholds)`
205+
: `${attemptKind} (prompt, thresholds)`)
199206
: truncate(JSON.stringify(body), 1200)
200207
};
201208
const log = {
@@ -213,7 +220,9 @@ export class ApiClient {
213220
url,
214221
headers: this._sanitizeHeaders(headers),
215222
bodyPreview: (endpointType === 'groundingdino')
216-
? (attemptKind === 'retry-json' ? 'multipart (initial) -> retried JSON (image+prompt+thresholds)' : 'multipart/form-data (file, prompt, thresholds)')
223+
? (attemptKind.includes('retry-json')
224+
? `${attemptKind} (prompt, thresholds)`
225+
: `${attemptKind} (prompt, thresholds)`)
217226
: truncate(JSON.stringify(body), 1200)
218227
};
219228
const log = {
@@ -381,14 +390,22 @@ export class ApiClient {
381390
}
382391

383392
// Shape B: { detections: [{ x,y,width,height,confidence }] } in pixel units
393+
// Also support nested: { detections: [{ bbox: { x,y,width,height }, score, label }] }
384394
if (Array.isArray(serverResponse.detections)) {
385395
for (const d of serverResponse.detections) {
396+
const hasFlat = Number.isFinite(Number(d?.x)) || Number.isFinite(Number(d?.width));
397+
const bb = d?.bbox;
398+
const x = hasFlat ? Number(d.x || 0) : Number(bb?.x || 0);
399+
const y = hasFlat ? Number(d.y || 0) : Number(bb?.y || 0);
400+
const w2 = hasFlat ? Number(d.width || 0) : Number(bb?.width || 0);
401+
const h2 = hasFlat ? Number(d.height || 0) : Number(bb?.height || 0);
402+
const conf = (d.confidence != null) ? Number(d.confidence) : (d.score != null ? Number(d.score) : 0);
386403
boxes.push({
387-
x: Math.max(0, Math.round(d.x || 0)),
388-
y: Math.max(0, Math.round(d.y || 0)),
389-
width: Math.max(0, Math.round(d.width || 0)),
390-
height: Math.max(0, Math.round(d.height || 0)),
391-
confidence: Math.max(0, Math.min(1, Number(d.confidence || 0)))
404+
x: Math.max(0, Math.round(x || 0)),
405+
y: Math.max(0, Math.round(y || 0)),
406+
width: Math.max(0, Math.round(w2 || 0)),
407+
height: Math.max(0, Math.round(h2 || 0)),
408+
confidence: Math.max(0, Math.min(1, Number(conf || 0)))
392409
});
393410
}
394411
}

0 commit comments

Comments
 (0)