added extension codebase

Author: zweack

.gitignore

@@ -0,0 +1,3 @@
+node_modules
+package-lock.json
+dist

README.md

@@ -1,2 +1,26 @@
 # ImageClassifier
 A browser extension to classify images, runs locally within browser.
+
+## Installation
+
+### Building the extension
+1. Clone the repository:
+    ```bash
+    git clone https://github.com/zweack/ImageClassifier.git
+    cd ImageClassifier
+    ```
+2. Install dependencies:
+   ```bash
+   npm install
+   ```  
+3. Build the extension:
+   ```bash
+   npm run build
+   ```
+4. Load the extension in your browser:
+   - For Chrome / Edge:
+     1. Open `chrome://extensions/`
+     2. Enable "Developer mode"
+     3. Click "Load unpacked" and select the `dist` directory
+    - For Firefox:
+    Coming soon...

ext/package.json

@@ -0,0 +1,19 @@
+{
+    "name": "image-classifier",
+    "version": "1.0.0",
+    "description": "Classify images in browser",
+    "scripts": {
+        "build": "webpack --config webpack.config.js"
+    },
+    "dependencies": {
+        "onnxruntime-web": "^1.20.1"
+    },
+    "devDependencies": {
+        "ts-loader": "^9.5.0",
+        "typescript": "^5.3.3",
+        "webpack": "^5.99.9",
+        "webpack-cli": "^6.0.1",
+        "webpack-obfuscator": "^3.5.1",
+        "copy-webpack-plugin": "^13.0.0"
+    }
+}

ext/src/background.js

@@ -0,0 +1,90 @@
+import * as ort from 'onnxruntime-web';
+
+class ImageClassifierBackground {
+    constructor() {
+        this.labels = null;
+        this.session = null;
+        this.init();
+    }
+
+    async init() {
+        await this.loadModelAndLabels();
+        this.setupContextMenu();
+        this.setupListeners();
+    }
+
+    async loadModelAndLabels() {
+        if (!this.session) {
+            const modelUrl = chrome.runtime.getURL('model.onnx');
+            this.session = await ort.InferenceSession.create(modelUrl);
+        }
+        if (!this.labels) {
+            const labelsUrl = chrome.runtime.getURL('labels.json');
+            this.labels = await fetch(labelsUrl).then(r => r.json());
+        }
+    }
+
+    setupContextMenu() {
+        chrome.contextMenus.create({
+            id: 'classify-image',
+            title: 'Classify this image',
+            contexts: ['image']
+        });
+        chrome.runtime.onInstalled.addListener(() => {
+            chrome.contextMenus.create({
+                id: 'classify-image',
+                title: 'Classify this image',
+                contexts: ['image']
+            });
+        });
+
+        chrome.contextMenus.onClicked.addListener((info, tab) => {
+            if (info.menuItemId === 'classify-image' && info.srcUrl) {
+                chrome.tabs.sendMessage(tab.id, {
+                    action: 'classify-image',
+                    imageUrl: info.srcUrl
+                });
+            }
+        });
+    }
+
+    setupListeners() {
+        chrome.runtime.onMessage.addListener(async (message, sender, sendResponse) => {
+            if (message.action === 'run-inference' && message.tensorData && message.tensorShape) {
+                try {
+                    await this.loadModelAndLabels();
+                    const outputNames = this.session.outputNames;
+                    const tensor = new ort.Tensor('float32', new Float32Array(message.tensorData), message.tensorShape);
+                    const feeds = {};
+                    feeds[this.session.inputNames[0]] = tensor;
+                    const results = await this.session.run(feeds);
+                    const output = results[outputNames[0]].data;
+                    let maxScore = -Infinity;
+                    let maxIndex = -1;
+                    for (let i = 0; i < output.length; i++) {
+                        if (output[i] > maxScore) {
+                            maxScore = output[i];
+                            maxIndex = i;
+                        }
+                    }
+                    let labelRes = {
+                        label: this.labels[maxIndex],
+                        confidence: (1 / (1 + Math.exp(-maxScore)))
+                    };
+                    chrome.tabs.sendMessage(sender.tab.id, {
+                        action: 'inference-result',
+                        result: labelRes
+                    });
+                } catch (e) {
+                    chrome.tabs.sendMessage(sender.tab.id, {
+                        action: 'inference-result',
+                        error: e.toString()
+                    });
+                }
+            }
+        });
+    }
+}
+
+new ImageClassifierBackground();
+

ext/src/content.js

@@ -0,0 +1,91 @@
+class ImageClassifierContent {
+  constructor() {
+    this.listenForMessages();
+  }
+
+  listenForMessages() {
+    chrome.runtime.onMessage.addListener(async (request, sender, sendResponse) => {
+      if (request.action === 'classify-image' && request.imageUrl) {
+        try {
+          this.showNotification('Classifying image...', 'info');
+          // Fetch the image as a blob
+          const response = await fetch(request.imageUrl);
+          const blob = await response.blob();
+          const img = await createImageBitmap(blob);
+
+          // Preprocess image to [1, 3, 224, 224] Float32Array
+          const floatData = await this.preprocessImage(img);
+          // Send tensor data to background for inference
+          chrome.runtime.sendMessage({
+            action: 'run-inference',
+            tensorData: Array.from(floatData),
+            tensorShape: [1, 3, 224, 224]
+          });
+        } catch (e) {
+          this.showNotification('Error preparing image: ' + e, 'error');
+        }
+      } else if (request.action === 'inference-result') {
+        if (request.error) {
+          this.showNotification('Inference error: ' + request.error, 'error');
+        } else {
+          // Expecting result: { label: string, confidence: number }
+          const result = request.result;
+          if (result && result.label && typeof result.confidence === 'number') {
+            this.showNotification(`This appears to be: ${result.label} (${(result.confidence * 100).toFixed(2)}% confidence)`, 'success');
+          } else {
+            this.showNotification('Classification result: ' + JSON.stringify(result), 'success');
+          }
+        }
+      }
+    });
+  }
+
+  async preprocessImage(img) {
+    const canvas = new OffscreenCanvas(224, 224);
+    const ctx = canvas.getContext('2d');
+    ctx.drawImage(img, 0, 0, 224, 224);
+    const imageData = ctx.getImageData(0, 0, 224, 224).data;
+    const floatData = new Float32Array(1 * 3 * 224 * 224);
+    for (let i = 0; i < 224 * 224; i++) {
+      floatData[i] = imageData[i * 4] / 255.0; // R
+      floatData[i + 224 * 224] = imageData[i * 4 + 1] / 255.0; // G
+      floatData[i + 2 * 224 * 224] = imageData[i * 4 + 2] / 255.0; // B
+    }
+    return floatData;
+  }
+
+  showNotification(message, type) {
+    const old = document.getElementById('image-classifier-notification');
+    if (old) old.remove();
+    const div = document.createElement('div');
+    div.id = 'image-classifier-notification';
+    div.textContent = message;
+    div.style.position = 'fixed';
+    div.style.top = '32px';
+    div.style.right = '32px';
+    div.style.zIndex = 99999;
+    if (type === 'success') {
+      div.style.background = '#27ae60';
+    } else if (type === 'error') {
+      div.style.background = '#ff4d4f';
+    } else {
+      div.style.background = '#222';
+    }
+    div.style.color = '#fff';
+    div.style.padding = '16px 24px';
+    div.style.borderRadius = '8px';
+    div.style.boxShadow = '0 2px 12px rgba(0,0,0,0.2)';
+    div.style.fontSize = '16px';
+    div.style.fontFamily = 'sans-serif';
+    div.style.cursor = 'pointer';
+    div.style.transition = 'opacity 0.3s';
+    div.onclick = () => div.remove();
+    document.body.appendChild(div);
+    setTimeout(() => {
+      div.style.opacity = '0';
+      setTimeout(() => div.remove(), 300);
+    }, 4000);
+  }
+}
+
+new ImageClassifierContent();