SimpleChatTC:WebTools And Search - headers and search drops - js

Allow the web tools handshake helper to pass additional header
entries provided by its caller.

Make use of this to send a list of tag and id pairs wrt web search
tool. Which will be used to drop div's matching the specified id.

Author: hanishkvc

tools/server/public_simplechat/simplechat.js

@@ -1290,10 +1290,19 @@ class MultiChatUI {
  * The SEARCHWORDS keyword will get replaced by the actual user specified search words at runtime.
  */
 const SearchURLS = {
-    duckduckgo: "https://duckduckgo.com/html/?q=SEARCHWORDS",
-    bing: "https://www.bing.com/search?q=SEARCHWORDS", // doesnt seem to like google chrome clients in particular
-    brave: "https://search.brave.com/search?q=SEARCHWORDS",
-    google: "https://www.google.com/search?q=SEARCHWORDS", // doesnt seem to like any client in general
+    duckduckgo: {
+        'template': "https://duckduckgo.com/html/?q=SEARCHWORDS",
+        'drop': [ { 'tag': 'div', 'id': "header" } ]
+    },
+    bing: {
+        'template': "https://www.bing.com/search?q=SEARCHWORDS", // doesnt seem to like google chrome clients in particular
+    },
+    brave: {
+        'template': "https://search.brave.com/search?q=SEARCHWORDS",
+    },
+    google: {
+        'template': "https://www.google.com/search?q=SEARCHWORDS", // doesnt seem to like any client in general
+    },
 }
 
 
@@ -1307,7 +1316,8 @@ class Me {
             enabled: true,
             proxyUrl: "http://127.0.0.1:3128",
             proxyAuthInsecure: "NeverSecure",
-            searchUrl: SearchURLS.duckduckgo,
+            searchUrl: SearchURLS.duckduckgo.template,
+            searchDrops: SearchURLS.duckduckgo.drop,
             toolNames: /** @type {Array<string>} */([]),
             /**
              * Control the length of the tool call result data returned to ai after tool call.

tools/server/public_simplechat/toolweb.mjs

@@ -49,15 +49,18 @@ function bearer_transform() {
  * @param {string} chatid
  * @param {string} toolcallid
  * @param {string} toolname
- * @param {any} obj
+ * @param {any} objSearchParams
  * @param {string} path
+ * @param {any} objHeaders
  */
-async function proxyserver_get_anyargs(chatid, toolcallid, toolname, obj, path) {
+async function proxyserver_get_anyargs(chatid, toolcallid, toolname, objSearchParams, path, objHeaders={}) {
     if (gToolsWorker.onmessage != null) {
-        let params = new URLSearchParams(obj)
+        let params = new URLSearchParams(objSearchParams)
         let newUrl = `${get_gme().tools.proxyUrl}/${path}?${params}`
+        let headers = new Headers(objHeaders)
         let btoken = await bearer_transform()
-        fetch(newUrl, { headers: { 'Authorization': `Bearer ${btoken}` }}).then(resp => {
+        headers.append('Authorization', `Bearer ${btoken}`)
+        fetch(newUrl, { headers: headers}).then(resp => {
             if (!resp.ok) {
                 throw new Error(`${resp.status}:${resp.statusText}`);
             }
@@ -256,7 +259,8 @@ function searchwebtext_run(chatid, toolcallid, toolname, obj) {
         searchUrl = searchUrl.replace("SEARCHWORDS", encodeURIComponent(obj.words));
         delete(obj.words)
         obj['url'] = searchUrl
-        return proxyserver_get_anyargs(chatid, toolcallid, toolname, obj, 'urltext');
+        let headers = { 'Search-Drops': get_gme().tools.searchDrops }
+        return proxyserver_get_anyargs(chatid, toolcallid, toolname, obj, 'urltext', headers);
     }
 }