diff --git a/.github/workflows/checklink.yml b/.github/workflows/checklink.yml index d9bcc164f..54001eb1d 100644 --- a/.github/workflows/checklink.yml +++ b/.github/workflows/checklink.yml @@ -37,6 +37,9 @@ jobs: HUGO_ENVIRONMENT: production HUGO_ENV: production run: hugo --environment GitHubPages -d $GITHUB_WORKSPACE/dist --buildFuture + - name: Generate Search index + run: | + node ./assets/js/generate-lunr-index.js $GITHUB_WORKSPACE/dist - name: Test HTML uses: wjdp/htmltest-action@master with: diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml index b62b3cdea..b8e4d2a34 100644 --- a/.github/workflows/hugo.yml +++ b/.github/workflows/hugo.yml @@ -63,6 +63,9 @@ jobs: HUGO_ENV: production run: | hugo --environment GitHubPages + - name: Generate Search index + run: | + node ./assets/js/generate-lunr-index.js - name: Upload artifact uses: actions/upload-pages-artifact@v3 with: diff --git a/.gitignore b/.gitignore index cb8937b14..4c46e5463 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ content/en/docs/latest/ content/static/latest/ content/static/**/*.dtmp content/static/**/*.bkp -content/static/**/*.crswap \ No newline at end of file +content/static/**/*.crswap +content/static/lunr-index.json \ No newline at end of file diff --git a/assets/js/generate-lunr-index.js b/assets/js/generate-lunr-index.js new file mode 100644 index 000000000..51519fb9e --- /dev/null +++ b/assets/js/generate-lunr-index.js @@ -0,0 +1,54 @@ +const fs = require('fs'); +const lunr = require('lunr'); + +/** + * This script is used to generate a lunr index from the offline-search-index.json file. + * Hugo must be built before running this script, as jt requires the offline-search-index.json file to have been generated. + * + * The script will output a lunr-index.json file in the content/static directory and the docs directory. + */ + +const args = process.argv.slice(2); + +// Arguments should only be provided from a pipeline build. +const isFromPipeline = args[0] !== undefined; + +const source = isFromPipeline + ? args[0] + : "./docs"; + +const destination = isFromPipeline + ? `${args[0]}` + : "./docs"; + +const data = JSON.parse(fs.readFileSync(`${source}/offline-search-index.json`)); + +const idx = lunr(function () { + this.ref('ref'); + this.field('title', { boost: 5 }); + this.field('categories', { boost: 3 }); + this.field('tags', { boost: 3 }); + this.field('description', { boost: 2 }); + this.field('body'); + + data.forEach((doc) => { + if (doc + && doc.ref !== undefined + && !doc.ref.includes('/_shared/') + ) { + this.add(doc); + } + }); +}); + +if (!isFromPipeline) { + fs.writeFileSync(`./content/static/lunr-index.json`, JSON.stringify(idx)); +} + +fs.writeFileSync(`${destination}/lunr-index.json`, JSON.stringify(idx)); + +// check if file got created +if (!fs.existsSync(`${destination}/lunr-index.json`)) { + console.error('Failed to create lunr index, hugo must be build using `hugo` command before running this script.'); + process.exit(1); +} \ No newline at end of file diff --git a/assets/js/offline-search.js b/assets/js/offline-search.js index 6f99da65a..7151cf686 100644 --- a/assets/js/offline-search.js +++ b/assets/js/offline-search.js @@ -1,4 +1,10 @@ -// Adapted from code by Matt Walters https://www.mattwalters.net/posts/hugo-and-lunr/ + +/** + * This script is used for the offline search feature. + * It calls the worker.js to generate the index and search the index. + * + * Adapted from code by Matt Walters https://www.mattwalters.net/posts/hugo-and-lunr/ + */ (function ($) { 'use strict'; @@ -6,10 +12,6 @@ $(document).ready(function () { const $searchInput = $('.td-search-input'); - // - // Options for popover - // - $searchInput.data('html', true); $searchInput.data('placement', 'bottom'); $searchInput.data( @@ -17,192 +19,137 @@ '' ); - // - // Register handler - // + let worker = null; - $searchInput.on('change', (event) => { - render($(event.target)); + if (window.Worker) { + worker = new Worker('/js/worker.js'); + const url = '/lunr-index.json'; - // Hide keyboard on mobile browser - $searchInput.blur(); - }); + worker.postMessage({ + type: 'init', + lunrIndexUrl: url, + rawIndexUrl: $searchInput.data('offline-search-index-json-src') + }); - // Prevent reloading page by enter key on sidebar search. - $searchInput.closest('form').on('submit', () => { - return false; - }); + worker.onerror = function (error) { + console.error('Error in worker:', error); + }; + } + + let currentTarget = null; + + worker.onmessage = function (event) { + if (event.data.type === 'search') { + const docs = event.data.docs; + const $html = $('
'); + + $html.append( + $('
') + .css({ + display: 'flex', + justifyContent: 'space-between', + marginBottom: '1em', + }) + .append( + $('') + .text('Search results') + .css({ fontWeight: 'bold' }) + ) + .append( + $('') + .addClass('fas fa-times search-result-close-button') + .css({ + cursor: 'pointer', + }) + ) + ); - // - // Lunr - // - - let idx = null; // Lunr index - const resultDetails = new Map(); // Will hold the data for the search results (titles and summaries) - - // Set up for an Ajax call to request the JSON data file that is created by Hugo's build process - $.ajax($searchInput.data('offline-search-index-json-src')).then( - (data) => { - idx = lunr(function () { - this.ref('ref'); - - // If you added more searchable fields to the search index, list them here. - // Here you can specify searchable fields to the search index - e.g. individual toxonomies for you project - // With "boost" you can add weighting for specific (default weighting without boost: 1) - this.field('title', { boost: 5 }); - this.field('categories', { boost: 3 }); - this.field('tags', { boost: 3 }); - this.field('description', { boost: 2 }); - this.field('body'); - - const searchPath = $searchInput.data('search-path'); - - data.forEach((doc) => { - let docToAdd; - if (searchPath !== undefined && doc.ref.startsWith(searchPath)) { - docToAdd = doc; - } else if (searchPath === undefined) { - docToAdd = doc; + const $searchResultBody = $('
').css({ + maxHeight: `calc(100vh - ${currentTarget.offset().top - + $(window).scrollTop() + + 180 + }px)`, + overflowY: 'auto', + }); + $html.append($searchResultBody); + + if (docs === undefined || docs.size === 0) { + $searchResultBody.append( + $('

').text(`No results found for query "${event.data.query}"`) + ); + } else { + docs.forEach((doc, key) => { + if (doc === undefined) { + return; } - if (docToAdd - && docToAdd.ref !== undefined - && !docToAdd.ref.includes('/_shared/') - ) { - this.add(doc); + const href = + $searchInput.data('offline-search-base-href') + + key.replace(/^\//, ''); - resultDetails.set(doc.ref, { - title: doc.title, - excerpt: doc.excerpt, - }); - } + const $entry = $('

').addClass('mt-4').addClass('search-result'); + + $entry.append( + $('') + .addClass('d-block') + .css({ + fontSize: '1.2rem', + }) + .attr('href', href) + .text(doc.title) + ); + + $entry.append( + $('').addClass('d-block text-muted').text(key) + ); + + $entry.append($('

').text(doc.excerpt)); + + $searchResultBody.append($entry); + }); + } + + currentTarget.on('shown.bs.popover', () => { + $('.search-result-close-button').on('click', () => { + currentTarget.val(''); + currentTarget.trigger('change'); }); }); - $searchInput.trigger('change'); + currentTarget + .data('content', $html[0].outerHTML) + .popover('show'); } - ); + } const render = ($targetSearchInput) => { - // Dispose the previous result $targetSearchInput.popover('dispose'); - - // - // Search - // - - if (idx === null) { - return; - } + currentTarget = $targetSearchInput; const searchQuery = $targetSearchInput.val(); if (searchQuery === '') { return; } - const results = idx - .query((q) => { - const tokens = lunr.tokenizer(searchQuery.toLowerCase()); - tokens.forEach((token) => { - const queryString = token.toString(); - q.term(queryString, { - boost: 100, - }); - q.term(queryString, { - wildcard: - lunr.Query.wildcard.LEADING | - lunr.Query.wildcard.TRAILING, - boost: 10, - }); - q.term(queryString, { - editDistance: 2, - }); - }); - }) - .slice( - 0, - $targetSearchInput.data('offline-search-max-results') - ); - - // - // Make result html - // - - const $html = $('

'); - - $html.append( - $('
') - .css({ - display: 'flex', - justifyContent: 'space-between', - marginBottom: '1em', - }) - .append( - $('') - .text('Search results') - .css({ fontWeight: 'bold' }) - ) - .append( - $('') - .addClass('fas fa-times search-result-close-button') - .css({ - cursor: 'pointer', - }) - ) - ); - - const $searchResultBody = $('
').css({ - maxHeight: `calc(100vh - ${$targetSearchInput.offset().top - - $(window).scrollTop() + - 180 - }px)`, - overflowY: 'auto', + worker.postMessage({ + type: 'search', + currentPath: window.location.pathname, + query: searchQuery, + maxResults: $targetSearchInput.data('offline-search-max-results') }); - $html.append($searchResultBody); - - if (results.length === 0) { - $searchResultBody.append( - $('

').text(`No results found for query "${searchQuery}"`) - ); - } else { - results.forEach((r) => { - const doc = resultDetails.get(r.ref); - const href = - $searchInput.data('offline-search-base-href') + - r.ref.replace(/^\//, ''); - - const $entry = $('

').addClass('mt-4').addClass('search-result'); - - $entry.append( - $('') - .addClass('d-block') - .css({ - fontSize: '1.2rem', - }) - .attr('href', href) - .text(doc.title) - ); - - $entry.append( - $('').addClass('d-block text-muted').text(r.ref) - ); - - $entry.append($('

').text(doc.excerpt)); + }; - $searchResultBody.append($entry); - }); - } + // Renders the search results when the input changes. + $searchInput.on('change', (event) => { + render($(event.target)); - $targetSearchInput.on('shown.bs.popover', () => { - $('.search-result-close-button').on('click', () => { - $targetSearchInput.val(''); - $targetSearchInput.trigger('change'); - }); - }); + // Hide keyboard on mobile browser + $searchInput.blur(); + }); - $targetSearchInput - .data('content', $html[0].outerHTML) - .popover('show'); - }; + // Prevent reloading page by enter key on sidebar search. + $searchInput.closest('form').on('submit', () => { + return false; + }); }); })(jQuery); diff --git a/assets/js/worker.js b/assets/js/worker.js new file mode 100644 index 000000000..e463940af --- /dev/null +++ b/assets/js/worker.js @@ -0,0 +1,99 @@ +/** + * This worker is responsible for loading the lunr index, searching the index and returning the results. + * + * It has 2 action types, 'init' and 'search'. + * + * 'init' action is used to load the lunr index and the raw index data. + * 'search' action is used to search the lunr index with the provided query. + */ + +if (typeof importScripts === 'function') { + importScripts('https://unpkg.com/lunr@2.3.8/lunr.min.js'); +} + +let idx; +const versionRegex = new RegExp("^\/docs\/([0-9\.]*|latest)\/"); +const resultDetails = new Map(); +let indexReadyPromise; + +self.onmessage = async function (event) { + if (event.data.type === 'init') { + // Initialize the lunr index and the raw index data. + indexReadyPromise = new Promise(async (resolve, reject) => { + try { + const rawIndex = await fetch(event.data.rawIndexUrl); + let json = await rawIndex.json(); + json.forEach((doc) => { + resultDetails.set(doc.ref, { + version: doc.version, + title: doc.title, + excerpt: doc.excerpt, + }); + }); + + const lunrIndex = await fetch(event.data.lunrIndexUrl); + json = await lunrIndex.json(); + idx = lunr.Index.load(json); + resolve(); + self.postMessage({ type: 'init', status: 'success' }); + } catch (error) { + reject(error); + self.postMessage({ type: 'init', status: 'error', message: error.message }); + } + }); + } else if (event.data.type === 'search') { + // Search the lunr index with the provided query. + try { + await indexReadyPromise; + const regexResults = versionRegex.exec(event.data.currentPath); + const version = regexResults + ? regexResults[0] + : undefined; + + const results = idx + .query((q) => { + const tokens = lunr.tokenizer(event.data.query.toLowerCase()); + tokens.forEach((token) => { + const queryString = token.toString(); + q.term(queryString, { + boost: 100, + }); + q.term(queryString, { + wildcard: lunr.Query.wildcard.LEADING | lunr.Query.wildcard.TRAILING, + boost: 10, + }); + q.term(queryString, { + editDistance: 2, + }); + }); + }); + + const docs = new Map(); + let count = 0; + + for (const result of results) { + if (count >= event.data.maxResults) { + break; + } + + if (resultDetails.get(result.ref) === undefined) { + continue; + } + + if (version === undefined) { + docs.set(result.ref, resultDetails.get(result.ref)); + } else if (result.ref.startsWith(version)) { + docs.set(result.ref, resultDetails.get(result.ref)); + } else { + continue; + } + + count++; + } + + self.postMessage({ type: 'search', status: 'success', query: event.data.query , docs: docs }); + } catch (error) { + self.postMessage({ type: 'search', status: 'error', message: 'Index not ready' }); + } + } +}; \ No newline at end of file diff --git a/layouts/docs/baseof.html b/layouts/docs/baseof.html index 64edd70ef..c8e57e24a 100644 --- a/layouts/docs/baseof.html +++ b/layouts/docs/baseof.html @@ -26,5 +26,7 @@ {{ partial "footer.html" . }}

{{ partial "scripts.html" . }} + {{ $script := resources.Get "js/worker.js"}} + \ No newline at end of file diff --git a/layouts/partials/search-input.html b/layouts/partials/search-input.html index bf974fcc6..733714315 100644 --- a/layouts/partials/search-input.html +++ b/layouts/partials/search-input.html @@ -6,8 +6,7 @@ {{ $path = index (findRE `^\/[^\/]*\/` .Page.RelPermalink 1) 0}} {{- end }} {{- $offlineSearchIndex := resources.Get "json/offline-search-index.json" | resources.ExecuteAsTemplate "offline-search-index.json" . }} - {{- /* Use `md5` as finger print hash function to shorten file name to avoid `file name too long` error. */}} - {{ $offlineSearchIndexFingerprint := $offlineSearchIndex | resources.Fingerprint "md5" }} + {{ $offlineSearchIndexFingerprint := $offlineSearchIndex }} =4" } }, + "node_modules/lunr": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.8.tgz", + "integrity": "sha512-oxMeX/Y35PNFuZoHp+jUj5OSEmLCaIH4KTFJh7a93cHBoFmpw2IoPs22VIz7vyO2YUnx2Tn9dzIwO2P/4quIRg==" + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -1822,6 +1830,11 @@ "chalk": "^2.0.1" } }, + "lunr": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.8.tgz", + "integrity": "sha512-oxMeX/Y35PNFuZoHp+jUj5OSEmLCaIH4KTFJh7a93cHBoFmpw2IoPs22VIz7vyO2YUnx2Tn9dzIwO2P/4quIRg==" + }, "merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", diff --git a/package.json b/package.json index f11e053ed..7aa76fb9b 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "Hugo theme for technical documentation.", "main": "none.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "build": "hugo && node ./assets/js/generate-lunr-index.js" }, "repository": { "type": "git", @@ -19,5 +19,8 @@ "devDependencies": { "autoprefixer": "^9.8.6", "postcss-cli": "^7.1.2" + }, + "dependencies": { + "lunr": "2.3.8" } }