Standardize rich search suggestions (ie icons)

[AskAlice]

Google implements this to chrome, and I've used google's implementation to build ontop of their work to provide rich search suggestions in the browser omnibox - https://github.com/AskAlice/search.emu.sh

However, a lot of people are going to be ditching chrome, including me, in the coming months as google enforces Manifest V3, and firefox does not actually support a similar standard from what I can tell. From what I can tell, there is no open standard for rich search suggestions.

here's an example:

I suggest opensearch suggestion specification includes not just text but also metadata-- suggestion text, suggestion URL, suggestion subtitle, suggestion description, and suggestion thumbnail

[Explosion-Scratch]

I would really love this as well, @AskAlice, that project also looks really cool, but trying to decipher the code is tricky for me. Could you provide me with an example response from your server that has an image, description, title and url?

[AskAlice]

@Explosion-Scratch I copied the response of google's own search suggestions, as to make it work with chrome.
The url for google's suggestions can be found at %localappdata%\Google\Chrome\User Data\Default\Web Data in the 'keywords' table. It substitutes in a bunch of variables that send session information to google, but if you just change the hostname to a local server, you can see what that looks like on your browser.

It responds with a txt file that the browser actually downloads instead of views, and it looks like this. Not exactly an open standard, but resembling opensearch's spec to a degree, with some added garbage text prefixing it

)]}'
["clerks",["clerks","clerks","clerks corner","clerks 3","clerks 2","clerkship","clerks and recorders","clerk's office","clerkship interview questions","clerks berserker"],["","","","","","","","","",""],[],{"google:suggesttype":["QUERY","ENTITY","QUERY","ENTITY","ENTITY","QUERY","QUERY","QUERY","QUERY","ENTITY"],"google:headertexts":[],"google:clientdata":[],"google:suggestsubtypes":[[512,433,131,355],[512,433,131],[512],[512,433,131],[512,433],[512,433],[512],[512,433,131,10],[512],[512]],"google:suggestdetail":[{},{"a":"1994 film","dc":"#424242","i":"https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQ_lJnB3jN0h3uM_-NEgzhN_RLuRMfEwBibOPkX2fQ&s=10","q":"gs_ssp=eJzj4tTP1TcwTM6oLDFg9GJLzkktyi4GADgeBf0","t":"Clerks","zae":"/m/01chyt"},{},{"a":"2022 film","dc":"#424242","i":"https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcTeQrF1QxRBBCwgV5a8O0HInz6nZZCnnZqPs_yX8UeNUdQ3KPC6xOZhMkRs&s=10","q":"gs_ssp=eJzj4tVP1zc0zDLLMyoqtMgwYPTiSM5JLcouVjAGAF4wB1w","t":"Clerks III","zae":"/g/11j6n2rq8h"},{"a":"2006 film","dc":"#a30202","i":"https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcSt4WtlCOfYQtXUTEP7aqaj73JEwambEEIJh3G9M78hlrLoYVhR9Wo44Tc&s=10","q":"gs_ssp=eJzj4tTP1TcwKahMrzJg9OJIzkktyi5WMAIARfAGZg","t":"Clerks II","zae":"/m/04pygz"},{},{},{},{},{"a":"Berserker — Song by Love Among Freaks","dc":"#424242","i":"https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcTQuVKceWu15zjJFsvHlGGRDI3PdC2Ey--6tuOPUFtqkgknB4G4nX7avyM&s=10","q":"gs_ssp=eJzj4tFP1zcsNjDNTcm1rDJg9BJIzkktyi5WSEotKgYyUosAo_AKyg","t":"clerks berserker","zae":"/g/1s05mdm9z"}],"google:suggestrelevance":[1300,1250,601,600,555,554,553,552,551,550]}]

prettified:

)]}'
[
  "clerks",
  [
    "clerks",
    "clerks",
    "clerks corner",
    "clerks 3",
    "clerks 2",
    "clerkship",
    "clerks and recorders",
    "clerk's office",
    "clerkship interview questions",
    "clerks berserker"
  ],
  [
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    ""
  ],
  [],
  {
    "google:suggesttype": [
      "QUERY",
      "ENTITY",
      "QUERY",
      "ENTITY",
      "ENTITY",
      "QUERY",
      "QUERY",
      "QUERY",
      "QUERY",
      "ENTITY"
    ],
    "google:headertexts": [],
    "google:clientdata": [],
    "google:suggestsubtypes": [
      [
        512,
        433,
        131,
        355
      ],
      [
        512,
        433,
        131
      ],
      [
        512
      ],
      [
        512,
        433,
        131
      ],
      [
        512,
        433
      ],
      [
        512,
        433
      ],
      [
        512
      ],
      [
        512,
        433,
        131,
        10
      ],
      [
        512
      ],
      [
        512
      ]
    ],
    "google:suggestdetail": [
      {},
      {
        "a": "1994 film",
        "dc": "#424242",
        "i": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQ_lJnB3jN0h3uM_-NEgzhN_RLuRMfEwBibOPkX2fQ&s=10",
        "q": "gs_ssp=eJzj4tTP1TcwTM6oLDFg9GJLzkktyi4GADgeBf0",
        "t": "Clerks",
        "zae": "/m/01chyt"
      },
      {},
      {
        "a": "2022 film",
        "dc": "#424242",
        "i": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcTeQrF1QxRBBCwgV5a8O0HInz6nZZCnnZqPs_yX8UeNUdQ3KPC6xOZhMkRs&s=10",
        "q": "gs_ssp=eJzj4tVP1zc0zDLLMyoqtMgwYPTiSM5JLcouVjAGAF4wB1w",
        "t": "Clerks III",
        "zae": "/g/11j6n2rq8h"
      },
      {
        "a": "2006 film",
        "dc": "#a30202",
        "i": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcSt4WtlCOfYQtXUTEP7aqaj73JEwambEEIJh3G9M78hlrLoYVhR9Wo44Tc&s=10",
        "q": "gs_ssp=eJzj4tTP1TcwKahMrzJg9OJIzkktyi5WMAIARfAGZg",
        "t": "Clerks II",
        "zae": "/m/04pygz"
      },
      {},
      {},
      {},
      {},
      {
        "a": "Berserker — Song by Love Among Freaks",
        "dc": "#424242",
        "i": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcTQuVKceWu15zjJFsvHlGGRDI3PdC2Ey--6tuOPUFtqkgknB4G4nX7avyM&s=10",
        "q": "gs_ssp=eJzj4tFP1zcsNjDNTcm1rDJg9BJIzkktyi5WSEotKgYyUosAo_AKyg",
        "t": "clerks berserker",
        "zae": "/g/1s05mdm9z"
      }
    ],
    "google:suggestrelevance": [
      1300,
      1250,
      601,
      600,
      555,
      554,
      553,
      552,
      551,
      550
    ]
  }
]

[Explosion-Scratch]

Thanks so much! So a .google:suggestdetail[] object's keys are like this, right:

a: Description
dc: Color
i: Image URL
q: Seems to be extra query parameters?
t: Title (alternative to what's displayed in the omnibox after selecting the item, which is contained in the first array)
zae: No idea

• And google:suggesttype must be ENTITY to display rich data (CALCULATOR seems to be another one)
• google:suggestsubtypes? Is this ok to leave out?

I totally understand that you didn't create this schema and may not know the answers to all of these questions, but it'd be greatly appreciated if you knew what google:suggestsubtypes, the zae key, and the other google:suggesttypes that are available.

[Explosion-Scratch]

Thanks!

[AskAlice]

Thanks so much! So a .google:suggestdetail[] object's keys are like this, right:

a: Description dc: Color i: Image URL q: Seems to be extra query parameters? t: Title (alternative to what's displayed in the omnibox after selecting the item, which is contained in the first array) zae: No idea

• And google:suggesttype must be ENTITY to display rich data (CALCULATOR seems to be another one)
• google:suggestsubtypes? Is this ok to leave out?

I totally understand that you didn't create this schema and may not know the answers to all of these questions, but it'd be greatly appreciated if you knew what google:suggestsubtypes, the zae key, and the other google:suggesttypes that are available.

That's the gist of it. More can be learned by looking at the chromium source code.
There are some icons built into the browser, I think the suggestdetail has an ansb property that can be in place of an image, reverse proxy your search engine through google's suggest url and you can find examples of that if you look up weather or stock symbols, it's an enum with values like this

1: Bookmark
2: up/down
3: Google Logo
4: Google Logo
5: Google Logo
6: Weather
7: Translation
8: Blank
9: Blank
10: Circular Arrows

I'll make an RFC for an open standard similar to this, though these browser specific icons would have to go, as well as google's zae and q

[AskAlice]

Thanks!

happy to share your source?

[Explosion-Scratch]

Thanks!

happy to share your source?

Of course! Also made an icon search via Iconify, which renders the icons as previews!

Repl.it source: https://replit.com/@ExplosionScratc/opensearchtest#index.js (Go to this page then hit activate in chrome search engines to use)

Code (Node.js with express)

const express = require('express');
const sharp = require("sharp");
require("isomorphic-fetch")

const BASE_URL = `https://opensearchtest.explosionscratc.repl.co/`;

const app = express();
app.use(express.json())

app.get("/", (req, res) => {
	res.sendFile(`${__dirname}/index.html`);
})

app.get("/os.xml", (req, res) => {
	res.sendFile(`${__dirname}/os.xml`)
})

app.get("/suggest/:term", async (req, res) => {
	console.log({ t: req.params.term })
	let term = req.params.term;

	let s = [{
		name: req.params.term,
		title: `Search for "${req.params.term}", Random string: ${Math.random().toString(36).slice(2)}`,
		description: "https://github.com/explosion-scratch",
		favicon: `https://api.iconify.design/ph:activity-fill.svg?color=red`,
		score: 0,
	}]

	if (term.startsWith("icon ")) {
		term = term.toLowerCase().trim();

		let icons = await fetch(`https://api.iconify.design/search?query=${encodeURIComponent(term.replace("icon ", ""))}&limit=3`).then(r => r.json()).then(j => j.icons.slice(0, 3));
		console.log(icons);
		s = icons.map(i => ({
			name: 'icon ' + i,
			score: 0,
			title: `'${i}' icon`,
			favicon: `${BASE_URL}icon/${i}`,
			description: `'${i}' icon from iconify`,
		}))
		console.log(s);
	}
	let a = getSuggestions(req.params.term, s);
	console.log(a);
	return res.json(a)
})

app.get("/icon/:icon", async (req, res) => {
	if (!/[a-z-]+\:[a-z+-]/.test(req.params.icon)) {
		res.status(404).json("Not found");
		return;
	}
	let svg = await fetch(`https://api.iconify.design/${req.params.icon}.svg?color=#888`).then(r => r.text());
	if (!svg.startsWith("<svg")) {
		return res.status(404).json("Not found")
	}
	let png = await sharp(Buffer.from(svg)).resize(256).png().toBuffer();
	res.writeHead(200, {
		'Content-Length': Buffer.byteLength(png),
		'Content-Type': 'image/png'
	})
	res.end(png);
})

app.get("/s/:thing", (req, res) => {
	res.redirect(`https://google.com/search?q=${encodeURIComponent(req.params.thing)}`)
})

app.all('*', (req, res) => {
	console.log(req.method, req.body, req.params, req.query, req.url)
	res.json({
		error: "test"
	})
});

app.listen(3000, () => {
	console.log('server started');
});

function getSuggestions(search, suggestions) {
	/*
	Suggestions is an array of items like so:
	{
		name: String,
		title: String,
		description: String,
		favicon: String,
		score?: Int,
		color?: String,
		subTypes?: String[],
		type: "ENTITY" | "CALCULATOR",
	}
	*/
	const isJSON = false;
	const suggestType = isJSON ? "suggestType" : "google:suggesttype";
	const suggestSubtypes = isJSON ? "suggestSubtypes" : "google:suggestsubtypes";
	const suggestDetail = isJSON ? "suggestDetail" : "google:suggestdetail";
	const suggestRelevance = isJSON
		? "suggestRelevance"
		: "google:suggestrelevance";
	const verbatimrelevance = isJSON
		? "verbatimrelevance"
		: "google:verbatimrelevance";
	const headerTexts = isJSON ? "headerTexts" : "google:headertexts";
	const clientData = isJSON ? "clientData" : "google:clientdata";
	let thing = suggestions.map((s) => {
		return {
			suggestion: s.name,
			[`${suggestType}`]: s.type || "ENTITY",
			[`${suggestSubtypes}`]: s.subTypes || ["thing"],
			[`${suggestDetail}`]: {
				a: s.description,
				dc: s.color || "#424242",
				i: s.favicon,
				q: "",
				t: s.title,
			},
			[`${suggestRelevance}`]: 99999 + s.score,
		};
	});
	let suggest = [
		[],
		[],
		[],
		{
			[suggestType]: [],
			[suggestSubtypes]: [],
			[suggestRelevance]: [],
			[suggestDetail]: [],
			"google:headertexts": [],
			"google:clientdata": [],
		},
	];
	for (let i of thing) {
		suggest[0].push(i.suggestion);
		suggest[1].push(
			i[suggestType] !== "ENTITY"
				? i[suggestType].slice(1) + i[suggestType].toLowerCase().slice(1)
				: ""
		);
		suggest[3][suggestType].push(i[suggestType]);
		suggest[3][suggestSubtypes].push(i[suggestSubtypes]);
		suggest[3][suggestRelevance].push(i[suggestRelevance]);
		suggest[3][suggestDetail].push(i[suggestDetail]);
	}
	return [search, ...suggest];
}

By far the most useful part of that though is the function which I made that takes an array of suggestions and turns it into the stuff that browsers expect in return:

function getSuggestions(search, suggestions) {
	/*
	Suggestions is an array of items like so:
	{
		name: String,
		title: String,
		description: String,
		favicon: String,
		score?: Int,
		color?: String,
		subTypes?: String[],
		type: "ENTITY" | "CALCULATOR",
	}
	*/
	const isJSON = false;
	const suggestType = isJSON ? "suggestType" : "google:suggesttype";
	const suggestSubtypes = isJSON ? "suggestSubtypes" : "google:suggestsubtypes";
	const suggestDetail = isJSON ? "suggestDetail" : "google:suggestdetail";
	const suggestRelevance = isJSON
		? "suggestRelevance"
		: "google:suggestrelevance";
	const verbatimrelevance = isJSON
		? "verbatimrelevance"
		: "google:verbatimrelevance";
	const headerTexts = isJSON ? "headerTexts" : "google:headertexts";
	const clientData = isJSON ? "clientData" : "google:clientdata";
	let thing = suggestions.map((s) => {
		return {
			suggestion: s.name,
			[`${suggestType}`]: s.type || "ENTITY",
			[`${suggestSubtypes}`]: s.subTypes || ["thing"],
			[`${suggestDetail}`]: {
				a: s.description,
				dc: s.color || "#424242",
				i: s.favicon,
				q: "",
				t: s.title,
			},
			[`${suggestRelevance}`]: 99999 + s.score,
		};
	});
	let suggest = [
		[],
		[],
		[],
		{
			[suggestType]: [],
			[suggestSubtypes]: [],
			[suggestRelevance]: [],
			[suggestDetail]: [],
			"google:headertexts": [],
			"google:clientdata": [],
		},
	];
	for (let i of thing) {
		suggest[0].push(i.suggestion);
		suggest[1].push(
			i[suggestType] !== "ENTITY"
				? i[suggestType].slice(1) + i[suggestType].toLowerCase().slice(1)
				: ""
		);
		suggest[3][suggestType].push(i[suggestType]);
		suggest[3][suggestSubtypes].push(i[suggestSubtypes]);
		suggest[3][suggestRelevance].push(i[suggestRelevance]);
		suggest[3][suggestDetail].push(i[suggestDetail]);
	}
	return [search, ...suggest];
}

[AskAlice]

looks quite familiar. My implementation was quite similar. I used string templating to dynamically name the keys (ie suggestType) based on whether or not it's a google result.

    const googleRes = {
      [`${suggestType}`]: [],
      [`${headerTexts}`]: [],
      [`${clientData}`]: [],
      [`${suggestSubtypes}`]: [],
      [`${suggestDetail}`]: [],
      [`${suggestRelevance}`]: [],
    };


    results.sort((a, b) => (a.relevance > b.relevance ? -1 : b.relevance > a.relevance ? 1 : 0));
    // console.log(searchFormat);
    console.log(JSON.stringify(results, null, 2));
    // return results;
    if (request?.query?.type === 'json' || request?.query?.format === 'json') return results;
    const searchFormat: Array<any> = ['', [], [], []];
    searchFormat[0] = request.query.q;
    results.forEach((res) =>
      Object.entries(res).forEach(([k, v], i) => (k === 'suggestion' ? searchFormat[1].push(v) && searchFormat[2].push('') : googleRes[k].push(v)))
    );
    ```
    re: this issue, I'm working on a draft for this RFC

[Explosion-Scratch]

[`${suggestType}`]: [],

Just a small note, you can just do [suggestType] as `${suggestType}` === suggestType.toString()