feat: add grounding dino support

Author: jan-beranek

README.md

@@ -38,6 +38,43 @@ npx http-server app -p 8080
   }
   ```
 
+### GroundingDINO Support (Optional)
+
+Besides LLM providers, the app can call a local GroundingDINO server for phrase‑grounded object detection.
+
+- Add a model and choose Endpoint Type: `GroundingDINO`.
+- Set Base URL to your detection endpoint (example server below serves `/groundingdino/detect`).
+- Adjust `Box thr` and `Text thr` as desired.
+
+Example server is included in `server/`:
+
+1) Create venv and install requirements, then install GroundingDINO from source.
+2) Set `GROUNDING_DINO_CONFIG_PATH` and `GROUNDING_DINO_WEIGHTS_PATH` env vars.
+3) Run: `uvicorn server.groundingdino_api:app --port 8001`
+4) In the app, set Base URL to `http://localhost:8001/groundingdino/detect` and test.
+
+The SPA adapts common GroundingDINO server responses into its canonical JSON for overlay rendering and CSV exports. It supports:
+- Pixel-space detections: `{ width, height, detections: [{ x,y,width,height,confidence }] }`
+- Label Studio–style results like your remote service returns: `{ results: [{ result: [{ type: 'rectanglelabels', value: { x,y,width,height,score } }], score }] }` with normalized [0..1] coordinates (scaled to pixels using the input image size).
+
+Remote example (already running):
+- Endpoint: `https://dino.d2.wopee.io/predict`
+- Configure a model with:
+  - Endpoint Type: `GroundingDINO`
+  - Base URL: `https://dino.d2.wopee.io/predict`
+  - Model (label): e.g. `GroundingDINO`
+  - Box thr: `0.35` • Text thr: `0.25`
+
+The client uploads the image as multipart/form-data with fields `file`, `prompt`, `box_threshold`, `text_threshold`, matching this curl you used:
+
+```
+curl -i -X POST "https://dino.d2.wopee.io/predict" \
+  -F "file=@Downloads/screenshot.png" \
+  -F "prompt=button" \
+  -F "box_threshold=0.35" \
+  -F "text_threshold=0.25"
+```
+
 ## Browser Support
 
 - Designed for evergreen browsers (ES modules). Uses `createImageBitmap({ imageOrientation: 'from-image' })` to respect EXIF rotation when supported, with a safe fallback.

app/index.html

@@ -13,6 +13,9 @@ <h1>LLM Visual Evaluator</h1>
   </header>
 
   <main class="layout" role="main">
+    <div id="corsWarning" style="display:none; margin:8px 8px 0 8px; padding:8px; border:1px solid #e9b34f; background:#fff8e6; color:#8a6100; border-radius:6px;">
+      Running from file:// has origin "null" and most APIs block it. Please serve the app over http(s), e.g. <code>python3 -m http.server -d app 8080</code>, then open <code>http://localhost:8080</code>.
+    </div>
     <!-- Inputs Panel (Left) -->
     <section class="panel inputs" aria-label="Inputs panel">
       <div class="section-block">

app/src/components/model-tabs.js

@@ -173,19 +173,19 @@ export class ModelTabs {
                 <input data-field="color" type="text" value="${cfg.color}"/>
               </div>
               <div>
-                <label>Model ID</label>
+                <label data-el="modelLabel">Model ID</label>
                 <input data-field="model" type="text" value="${cfg.model}" placeholder="gpt-4o-mini"/>
               </div>
               <div class="full">
                 <label>Base URL</label>
                 <input data-field="baseURL" type="text" value="${cfg.baseURL}" placeholder="https://api.example.com/v1"/>
               </div>
               <div>
-                <label>API Key</label>
+                <label data-el="apiKeyLabel">API Key</label>
                 <input data-field="apiKey" type="password" value="${cfg.apiKey || ''}" placeholder="sk-..."/>
               </div>
               <div>
-                <label>API Version</label>
+                <label data-el="apiVersionLabel">API Version</label>
                 <input data-field="apiVersion" type="text" value="${cfg.apiVersion || ''}" placeholder="2024-08-01-preview"/>
               </div>
             </div>
@@ -195,7 +195,7 @@ export class ModelTabs {
           <div class="model-section">
             <div class="model-grid wide">
               <div>
-                <label>Max tokens</label>
+                <label data-el="maxTokensLabel">Max tokens</label>
                 <input data-field="maxTokens" type="number" value="${cfg.maxTokens ?? 2048}"/>
               </div>
               <div>
@@ -211,6 +211,7 @@ export class ModelTabs {
               <div class="btn-group" role="group" aria-label="Endpoint type">
                 <button type="button" class="btn endpoint-btn ${cfg.endpointType==='chat' ? 'active':''}" data-endpoint="chat" aria-pressed="${cfg.endpointType==='chat'}">Chat</button>
                 <button type="button" class="btn endpoint-btn ${cfg.endpointType==='responses' ? 'active':''}" data-endpoint="responses" aria-pressed="${cfg.endpointType==='responses'}">Responses</button>
+                <button type="button" class="btn endpoint-btn ${cfg.endpointType==='groundingdino' ? 'active':''}" data-endpoint="groundingdino" aria-pressed="${cfg.endpointType==='groundingdino'}">GroundingDINO</button>
               </div>
               <div class="inline-field endpoint-field" data-endpoint="chat" style="display:${cfg.endpointType==='chat' ? 'flex':'none'}">
                 <label>Temperature</label>
@@ -225,6 +226,12 @@ export class ModelTabs {
                   <option value="high" ${cfg.reasoningEffort==='high'?'selected':''}>high</option>
                 </select>
               </div>
+              <div class="inline-field endpoint-field" data-endpoint="groundingdino" style="display:${cfg.endpointType==='groundingdino' ? 'flex':'none'}; gap:8px;">
+                <label>Box thr</label>
+                <input data-field="dinoBoxThreshold" type="number" step="0.01" min="0" max="1" value="${cfg.dinoBoxThreshold ?? 0.35}" style="width:90px"/>
+                <label>Text thr</label>
+                <input data-field="dinoTextThreshold" type="number" step="0.01" min="0" max="1" value="${cfg.dinoTextThreshold ?? 0.25}" style="width:90px"/>
+              </div>
             </div>
           </div>
         </div>
@@ -246,12 +253,18 @@ export class ModelTabs {
     let currentEndpointType = cfg.endpointType || 'chat';
     const baseURL = card.querySelector('input[data-field="baseURL"]');
     const model = card.querySelector('input[data-field="model"]');
+    const modelLabelEl = card.querySelector('[data-el="modelLabel"]');
     const apiVersion = card.querySelector('input[data-field="apiVersion"]');
+    const apiVersionLabelEl = card.querySelector('[data-el="apiVersionLabel"]');
     const reasoningEffort = card.querySelector('select[data-field="reasoningEffort"]');
     const key = card.querySelector('input[data-field="apiKey"]');
+    const apiKeyLabelEl = card.querySelector('[data-el="apiKeyLabel"]');
     const temp = card.querySelector('input[data-field="temperature"]');
     const maxTok = card.querySelector('input[data-field="maxTokens"]');
+    const maxTokensLabelEl = card.querySelector('[data-el="maxTokensLabel"]');
     const timeout = card.querySelector('input[data-field="timeoutMs"]');
+    const dinoBoxThreshold = card.querySelector('input[data-field="dinoBoxThreshold"]');
+    const dinoTextThreshold = card.querySelector('input[data-field="dinoTextThreshold"]');
 
     const headersTa = card.querySelector('textarea[data-field="extraHeaders"]');
     const logEl = card.querySelector('[data-log]');
@@ -311,6 +324,38 @@ export class ModelTabs {
     color.addEventListener('focus', showColorPopover);
     color.addEventListener('blur', () => { /* keep open for interactions; closed by outside click */ });
 
+    const updateEndpointNonApplicableUI = () => {
+      const isDino = currentEndpointType === 'groundingdino';
+      if (model) {
+        model.disabled = isDino;
+        model.placeholder = isDino ? '(not used for DINO)' : 'gpt-4o-mini';
+        if (isDino) {
+          const v = String(model.value || '');
+          if (!v || /(gpt|claude|qwen|llava|mini|vision)/i.test(v)) {
+            model.value = 'GroundingDINO';
+          }
+        }
+      }
+      if (modelLabelEl) {
+        modelLabelEl.textContent = isDino ? 'Display name (UI only)' : 'Model ID';
+      }
+      if (apiVersion) apiVersion.disabled = isDino;
+      if (apiVersionLabelEl) apiVersionLabelEl.textContent = isDino ? 'API Version (n/a)' : 'API Version';
+      if (maxTok) maxTok.disabled = isDino;
+      if (maxTokensLabelEl) maxTokensLabelEl.textContent = isDino ? 'Max tokens (n/a)' : 'Max tokens';
+      if (key) {
+        key.disabled = isDino;
+        key.placeholder = isDino ? '(not used for DINO)' : 'sk-...';
+      }
+      if (apiKeyLabelEl) apiKeyLabelEl.textContent = isDino ? 'API Key (n/a)' : 'API Key';
+      if (headersTa) {
+        headersTa.disabled = isDino;
+        headersTa.placeholder = isDino ? '(not used for DINO)' : '{"X-Org":"..."}';
+      }
+    };
+    // Initialize once
+    updateEndpointNonApplicableUI();
+
     const persist = () => {
       let extra = undefined;
       try {
@@ -333,6 +378,8 @@ export class ModelTabs {
         maxTokens: Number(maxTok.value),
         timeoutMs: Number(timeout.value),
         extraHeaders: extra,
+        dinoBoxThreshold: dinoBoxThreshold ? Number(dinoBoxThreshold.value) : undefined,
+        dinoTextThreshold: dinoTextThreshold ? Number(dinoTextThreshold.value) : undefined,
       };
       this.storage.updateModel(updated);
 
@@ -406,6 +453,8 @@ export class ModelTabs {
         endpointPanels.forEach(p => {
           p.style.display = (p.dataset.endpoint === currentEndpointType) ? 'flex' : 'none';
         });
+        // Update disabled/labels for non-applicable fields
+        updateEndpointNonApplicableUI();
         persist();
       });
     });
@@ -418,6 +467,8 @@ export class ModelTabs {
     timeout.addEventListener('input', persist);
     headersTa.addEventListener('input', persist);
     reasoningEffort.addEventListener('change', persist);
+    if (dinoBoxThreshold) dinoBoxThreshold.addEventListener('input', persist);
+    if (dinoTextThreshold) dinoTextThreshold.addEventListener('input', persist);
 
     return card;
   }