Merge pull request #2 from phreakocious/viewer

Major extension overhaul.. web viewer with live streaming from the extension.. Python graph builder

Author: phreakocious

.github/workflows/ci.yml

@@ -0,0 +1,21 @@
+name: CI
+
+on:
+  push:
+    branches: [master, manifest_v3]
+  pull_request:
+    branches: [master, manifest_v3]
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: npm
+      - run: npm ci
+      - run: npm audit
+      - run: npm run lint
+      - run: npm test

.gitignore

@@ -4,3 +4,5 @@
 *.pem
 *.sublime-*
 *.json
+node_modules/
+.venv/

dist/httpgraph.js

@@ -9,6 +9,28 @@
 
 const default_rest_port = "65444";
 const default_scrub_parameters = false;
+const default_collecting = true;
+const default_domain_include = "";
+const default_domain_exclude = "";
+
+// Request timing map — stores start times keyed by requestId
+const requestTimings = new Map();
+
+// Initiator map — stores the origin that triggered each request
+const requestInitiators = new Map();
+
+// Connected viewer ports for live streaming
+const viewerPorts = new Set();
+
+function broadcastToViewers(data) {
+    for (const port of viewerPorts) {
+        try {
+            port.postMessage(data);
+        } catch (e) {
+            viewerPorts.delete(port);
+        }
+    }
+}
 
 // hashing function courtesy of bryc https://github.com/bryc/code/blob/master/jshash/experimental/cyrb53.js
 const cyrb53 = (str, seed = 42) => {
@@ -23,22 +45,56 @@ const cyrb53 = (str, seed = 42) => {
     return 4294967296 * (2097151 & h2) + (h1 >>> 0);
 };
 
-function scrubber(match, p1, offset, string) {
+function scrubber(match, p1, _offset, _string) {
 	return "?SCRUBBED_hash=" + cyrb53(p1);
 }
 
+function updateBadge(collecting) {
+	if (collecting) {
+		chrome.action.setBadgeText({ text: "" });
+	} else {
+		chrome.action.setBadgeText({ text: "OFF" });
+		chrome.action.setBadgeBackgroundColor({ color: "#cc0000" });
+	}
+}
+
+function domainMatches(hostname, domainList) {
+	return domainList.some(domain => hostname === domain || hostname.endsWith("." + domain));
+}
+
 async function logResponse(details) {
     // Get settings from storage every time, as the service worker can be terminated.
     const items = await chrome.storage.local.get({
         rest_port: default_rest_port,
-        scrub_parameters: default_scrub_parameters
+        scrub_parameters: default_scrub_parameters,
+        collecting: default_collecting,
+        domain_include: default_domain_include,
+        domain_exclude: default_domain_exclude
     });
 
+    // If collection is paused, do nothing
+    if (!items.collecting) return;
+
     const url_backend = `http://127.0.0.1:${items.rest_port}/add_record`;
 
     // Avoid feedback loop and internal browser requests
  	if ( details.url.startsWith(url_backend) || details.tabId < 0 ) return;
 
+    // Domain filtering
+    try {
+        const hostname = new URL(details.url).hostname;
+        const includeList = items.domain_include.split("\n").map(s => s.trim().toLowerCase()).filter(Boolean);
+        const excludeList = items.domain_exclude.split("\n").map(s => s.trim().toLowerCase()).filter(Boolean);
+
+        if (includeList.length > 0) {
+            if (!domainMatches(hostname.toLowerCase(), includeList)) return;
+        } else if (excludeList.length > 0) {
+            if (domainMatches(hostname.toLowerCase(), excludeList)) return;
+        }
+    } catch (_e) {
+        // If URL parsing fails, proceed anyway
+    }
+
 	const headers = details.responseHeaders;
 	let data = {
 	  url: details.url,
@@ -49,6 +105,20 @@ async function logResponse(details) {
 	  type: details.type
 	};
 
+    // Compute request duration if we have a start time
+    const startTime = requestTimings.get(details.requestId);
+    if (startTime !== undefined) {
+        data.duration_ms = Math.round(details.timeStamp - startTime);
+        requestTimings.delete(details.requestId);
+    }
+
+    // Attach initiator origin if captured from onBeforeRequest
+    const initiator = requestInitiators.get(details.requestId);
+    if (initiator) {
+        data.initiator = initiator;
+        requestInitiators.delete(details.requestId);
+    }
+
 	for (const header of headers) {
         const headerName = header.name.toLowerCase();
 		if (headerName === 'content-length') {
@@ -90,17 +160,115 @@ async function logResponse(details) {
     } catch (error) {
         console.error("HTTP Graph Error: Could not send data to backend.", error);
     }
+
+    broadcastToViewers(finalData);
 }
 
+const requestFilter = { urls: [ "http://*/*", "https://*/*" ] };
+
+chrome.webRequest.onBeforeRequest.addListener(
+	(details) => {
+		requestTimings.set(details.requestId, details.timeStamp);
+		if (details.initiator && details.initiator !== "null") {
+			requestInitiators.set(details.requestId, details.initiator);
+		}
+	},
+	requestFilter
+);
+
 chrome.webRequest.onCompleted.addListener(
 	logResponse,
-	{ urls: [ "http://*/*", "https://*/*" ] },
+	requestFilter,
 	[ "responseHeaders" ]
 );
 
+chrome.webRequest.onBeforeRedirect.addListener(
+	async (details) => {
+		const items = await chrome.storage.local.get({
+			rest_port: default_rest_port,
+			collecting: default_collecting,
+			domain_include: default_domain_include,
+			domain_exclude: default_domain_exclude
+		});
+
+		if (!items.collecting) return;
+
+		const url_backend = `http://127.0.0.1:${items.rest_port}/add_record`;
+		if (details.url.startsWith(url_backend) || details.tabId < 0) return;
+
+		// Domain filtering on the source URL
+		try {
+			const hostname = new URL(details.url).hostname;
+			const includeList = items.domain_include.split("\n").map(s => s.trim().toLowerCase()).filter(Boolean);
+			const excludeList = items.domain_exclude.split("\n").map(s => s.trim().toLowerCase()).filter(Boolean);
+
+			if (includeList.length > 0) {
+				if (!domainMatches(hostname.toLowerCase(), includeList)) return;
+			} else if (excludeList.length > 0) {
+				if (domainMatches(hostname.toLowerCase(), excludeList)) return;
+			}
+		} catch (_e) {
+			// If URL parsing fails, proceed anyway
+		}
+
+		const data = {
+			edge_type: "redirect",
+			url: details.url,
+			redirect_url: details.redirectUrl,
+			ts: details.timeStamp,
+			ip: details.ip,
+			method: details.method,
+			status: details.statusCode,
+			type: details.type
+		};
+
+		try {
+			await fetch(url_backend, {
+				method: 'POST',
+				headers: { 'Content-Type': 'application/json' },
+				body: JSON.stringify(data) + "\r\n"
+			});
+		} catch (error) {
+			console.error("HTTP Graph Error: Could not send redirect data to backend.", error);
+		}
+
+		broadcastToViewers(data);
+	},
+	requestFilter,
+	[ "responseHeaders" ]
+);
+
+chrome.webRequest.onErrorOccurred.addListener(
+	(details) => {
+		requestTimings.delete(details.requestId);
+		requestInitiators.delete(details.requestId);
+	},
+	requestFilter
+);
+
 chrome.runtime.onInstalled.addListener(() => {
 	chrome.storage.local.set({
 		rest_port: default_rest_port,
-		scrub_parameters: default_scrub_parameters
+		scrub_parameters: default_scrub_parameters,
+		collecting: default_collecting,
+		domain_include: default_domain_include,
+		domain_exclude: default_domain_exclude
+	});
+	updateBadge(default_collecting);
+});
+
+chrome.runtime.onStartup.addListener(async () => {
+	const items = await chrome.storage.local.get({ collecting: default_collecting });
+	updateBadge(items.collecting);
+});
+
+// ── Live Viewer Streaming ──
+chrome.runtime.onConnectExternal.addListener((port) => {
+	if (port.name !== "httpgraph-viewer") return;
+	console.log("HTTP Graph: Viewer connected");
+	viewerPorts.add(port);
+	port.onDisconnect.addListener(() => {
+		viewerPorts.delete(port);
+		console.log("HTTP Graph: Viewer disconnected");
 	});
-});
+});

dist/manifest.json

@@ -1,8 +1,8 @@
 {
     "manifest_version": 3,
     "name": "HTTP Graph Collector",
-    "version": "0.4",
-    "description": "Accompanies the HTTP Graph plugin for Gephi. Collects minimal HTTP and HTTPS header data and POSTs to a REST API as you browse.",
+    "version": "0.6",
+    "description": "Collects HTTP/HTTPS request metadata as you browse and streams it live to the HTTP Graph Viewer at nullphase.net/hg or POSTs to a localhost REST API. All data stays local to your browser — nothing is sent over the network.",
     "background":
     {
         "service_worker": "httpgraph.js"
@@ -22,19 +22,24 @@
     "action":
     {
         "default_title": "HTTP Graph Collector",
+        "default_popup": "popup/popup.html",
         "default_icon":
         {
             "48": "httpgraph-48x48.png"
         }
     },
+    "externally_connectable":
+    {
+        "matches":
+        [
+            "https://nullphase.net/*",
+            "http://localhost/*",
+            "http://127.0.0.1/*"
+        ]
+    },
     "icons":
     {
         "48": "httpgraph-48x48.png",
         "128": "httpgraph-128x128.png"
-    },
-    "options_ui":
-    {
-        "page": "options/options.html",
-        "open_in_tab": false
     }
 }

dist/options/options.html

@@ -21,12 +21,24 @@
   </p>
   <p>
     (JSON data will POST to: http://127.0.0.1:<b><span id=url_port>65444</span></b>/add_record)
-    <br />
+  </p>
+  <hr />
+  <p>
+    <label for="domain_include"><b>Only collect from these domains</b> (one per line):</label><br />
+    <textarea id="domain_include" rows="4" cols="40" placeholder="example.com&#10;another.org"></textarea>
+  </p>
+  <p>
+    <label for="domain_exclude"><b>Never collect from these domains</b> (one per line):</label><br />
+    <textarea id="domain_exclude" rows="4" cols="40" placeholder="ads.example.com&#10;tracker.net"></textarea>
+  </p>
+  <p><small>If the include list is non-empty, only matching domains are collected and the exclude list is ignored.</small></p>
+  <hr />
+  <p>
     <div id="status">&nbsp;</div>
   </p>
   <p style="float: left;"><button id="save">Save</button></p>
   <p style="float: right;">by <a href="https://twitter.com/phreakocious" target="_blank">@phreakocious</a></p>
   <div style="clear: both;"></div>
   <script src="options.js" type="text/javascript"></script>
 </body>
-</html>
+</html>

dist/options/options.js

@@ -1,26 +1,33 @@
 function save_options() {
   var rest_port = document.getElementById('rest_port').value
   var scrub_parameters = document.getElementById('scrub_parameters').checked
+  var domain_include = document.getElementById('domain_include').value
+  var domain_exclude = document.getElementById('domain_exclude').value
   chrome.storage.local.set({
     rest_port: rest_port,
-    scrub_parameters: scrub_parameters
+    scrub_parameters: scrub_parameters,
+    domain_include: domain_include,
+    domain_exclude: domain_exclude
   }, function() {
     var status = document.getElementById('status')
     status.textContent = 'Options saved.'
     setTimeout(function() {
       status.innerHTML = ' '
     }, 750)
-    chrome.extension.getBackgroundPage().window.location.reload()
   })
 }
 
 function restore_options() {
   chrome.storage.local.get({
-    rest_port,
-    scrub_parameters
+    rest_port: "65444",
+    scrub_parameters: false,
+    domain_include: "",
+    domain_exclude: ""
   }, function(items) {
     document.getElementById('rest_port').value = items.rest_port
     document.getElementById('scrub_parameters').checked = items.scrub_parameters
+    document.getElementById('domain_include').value = items.domain_include
+    document.getElementById('domain_exclude').value = items.domain_exclude
   })
 }