Merge branch 'migrate-search-to-hugo-and-pagefind'

With this topic branch, the server-side search is replaced by a
client-side search that uses Pagefind (https://pagefind.app/) to
generate a nicely-chunked search index that can be loaded on demand,
incrementally, and efficiently.

While Hugo takes about half a minute to run in my setup, building the
search index takes about three minutes, testament of the fact that
generating search indices is a much more complex task than rendering a
static site.

Signed-off-by: Johannes Schindelin <[email protected]>

Author: dscho

.github/workflows/ci.yml

@@ -7,10 +7,11 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: configure Hugo version
+    - name: configure Hugo and Pagefind version
       run: |
         set -x &&
         echo "HUGO_VERSION=$(sed -n 's/^ *hugo_version: *//p' <hugo.yml)" >>$GITHUB_ENV
+        echo "PAGEFIND_VERSION=$(sed -n 's/^ *pagefind_version: *//p' <hugo.yml)" >>$GITHUB_ENV
 
     - name: install Hugo ${{ env.HUGO_VERSION }}
       run: |
@@ -21,6 +22,9 @@ jobs:
     - name: run Hugo to build the pages
       run: hugo
 
+    - name: run Pagefind ${{ env.PAGEFIND_VERSION }} to build the search index
+      run: npx -y pagefind@${{ env.PAGEFIND_VERSION }} --site public
+
     - name: build tar archive
       run: cd public && tar czvf ../pages.tar.gz *
 
@@ -29,3 +33,17 @@ jobs:
       with:
         name: pages
         path: pages.tar.gz
+
+    - name: verify search results
+      run: |
+        set -x
+        res=0
+        for term in commit config log rev-parse
+        do
+          node ./script/run-pagefind.js "$term" | tee search.results >&2
+          grep "^1. /docs/git-$term\\.html" search.results || {
+            echo "::error::Search for $term failed to show the manual page first" >&2
+            res=1
+          }
+        done
+        exit $res

README.md

@@ -51,6 +51,19 @@ Side note: What _are_ "ugly URLs"? Hugo, by default, generates "pretty" URLs lik
 
 Pro-Tip: Do this in a sparse checkout that excludes large parts of `content/`, to speed up the rendering time.
 
+To test the site locally _with_ the search enabled, run this instead:
+
+    $ hugo
+    $ npx -y pagefind --site public
+    $ node script/serve-public.js
+
+You can also use Pagefind's built-in server (which will be running on http://127.0.0.1:1414), but again, you have to turn off "ugly URLs":
+
+    $ HUGO_UGLYURLS=false hugo
+    $ npx -y pagefind --site public --serve
+
+Note that running Pagefind will make the process about 7 times slower, and the site will not be re-rendered and live-reloaded in the browser when you change files in `content/` (unlike with `hugo serve -w`).
+
 ## Update manual pages
 
 (TODO!)
@@ -134,6 +147,10 @@ The [list of GUI clients](https://git-scm.com/downloads/guis) has been construct
 * https://gohugo.io/
 * https://gohugo.io/content-management/shortcodes/
 
+### Pagefind (client-side search)
+
+* https://pagefind.app/
+
 ## License
 
 The source code for the site is licensed under the MIT license, which you can find in

app/views/site/results.html.erb

@@ -98,6 +98,7 @@ var Search = {
   selectedIndex: 0,
 
   init: function() {
+    Search.displayFullSearchResults();
     Search.observeFocus();
     Search.observeTextEntry();
     Search.observeResultsClicks();
@@ -167,10 +168,78 @@ var Search = {
 
       if(term != Search.currentSearch) {
         Search.currentSearch = term;
-        $.get("/search", {search: term}, function(results) {
-          $("#search-results").html(results);
+        $("#search-results").html(`
+          <header> Search Results </header>
+          <table>
+            <tbody>
+              <tr class="show-all">
+               <td class="category"> &nbsp; </td>
+                <td class="matches">
+                  <ul>
+                    <li>
+                      <a class="highlight" id="show-results-label">Searching for <span id="search-term">&nbsp;</span>...</a>
+                    </li>
+                  </ul>
+                </td>
+              </tr>
+              <tr>
+                <td class="category"> &nbsp; </td>
+                <td class="matches">
+                  <ul>
+                    <li><button id="load-more-results">Loading</button></li>
+                  </ul>
+                </td>
+              </tr>
+            </tbody>
+          </table>
+        `);
+        $("#search-term").text(term);
+        this.initializeSearchIndex(async () => {
+          const results = await Search.pagefind.debouncedSearch(term);
+          if (results === null) return;
+          if (results.results.length === 0) {
+            $("#show-results-label").text("No matching pages found.");
+            return;
+          }
+          $("#show-results-label").text("Show all results...");
+          const loadButton = $("#load-more-results");
+          loadButton.text(`Loading ${
+            results.results.length < 2
+            ? "result"
+            : `${results.results.length} results`
+          }`);
+          loadButton.loading = false;
+
+          const chunkLength = 10;
+          let displayCount = 0;
+          const loadResultsChunk = () => {
+            if (loadButton.loading || displayCount >= results.results.length) return;
+
+            loadButton.loading = true;
+            const n = displayCount + chunkLength;
+            while (displayCount < n) {
+              const li = $("<li><a>&hellip;</a></li>");
+              li.insertBefore(loadButton);
+
+              // load the result lazily
+              (async () => {
+                const result = await results.results[displayCount].data();
+                li.html(`<a href = "${result.url}">${result.meta.title}</a>`);
+              })().catch(console.log);
+
+              if (++displayCount >= results.results.length) {
+                loadButton.remove();
+                return;
+              }
+            }
+            const remaining = results.results.length - displayCount;
+            loadButton.text(`Load ${remaining} more ${remaining < 2 ? "result" : "results"}`);
+            loadButton.loading = false;
+          };
+          loadResultsChunk();
+          loadButton.on("click", loadResultsChunk);
           Search.searching = false;
-        }, 'html');
+        });
       };
     }
     else {
@@ -186,8 +255,9 @@ var Search = {
     var link = $('#search-results a')[Search.selectedIndex];
     var url = $(link).attr('href');
     if(!url) {
-      var term = $('#search-text').val();
-      url = "/search/results?search=" + term;
+      const term = $('#search-text').val();
+      const language = document.querySelector("html")?.getAttribute("lang");
+      url = `search/results?search=${term}${language && `&language=${language}`}`;
     }
     window.location.href = url;
     selectedIndex = 0;
@@ -207,6 +277,55 @@ var Search = {
     $('form#search').switchClass("focus", "", 200);
     $('#search-results').fadeOut(0.2);
     Search.selectedIndex = 0;
+  },
+
+  getQueryValue: function(key) {
+    const query = window.location.search.substring(1);
+    const needle = `${key}=`;
+    return query
+      .split('&')
+      .filter(e => e.startsWith(needle))
+      .map(e => decodeURIComponent(e.substring(needle.length).replace(/\+/g, '%20')))
+      .pop();
+  },
+
+  initializeSearchIndex: function(callback) {
+    if (Search.pagefind) {
+      callback().catch(console.log);
+      return;
+    }
+    (async () => {
+      Search.pagefind = await import(`${baseURLPrefix}pagefind/pagefind.js`);
+      const options = {}
+      const language = this.getQueryValue('language');
+      if (language) options.language = language;
+      await Search.pagefind.options(options);
+      await Search.pagefind.init();
+      await callback();
+    })().catch(console.log);
+  },
+
+  displayFullSearchResults: function() {
+    if (!$("#search-div").length) return;
+
+    const language = this.getQueryValue('language');
+
+    const ui = new PagefindUI({
+      element: "#search-div",
+      showSubResults: true,
+      showImages: false,
+      language,
+      processResult: function (result) {
+        result.url = result.url.replace(/\.html$/, "")
+	return result
+      },
+    });
+
+    const searchTerm = this.getQueryValue('search');
+    if (searchTerm) {
+      $("#search-div input").val(searchTerm)
+      ui.triggerSearch(searchTerm);
+    }
   }
 }
 

assets/js/application.js

@@ -39,7 +39,7 @@ form#search {
   width: 262px;
   padding-left: 32px;
   @include background-image-2x($baseurl + "images/icons/search", 17px, 17px, 10px 50%);
-  background-color: #fcfcfa !important;
+  background-color: $main-bg !important;
   border: solid 1px #ceccc5;
   @include border-radius(20px);
   @include box-shadow(inset 0 1px 4px #ddd);

assets/sass/forms.scss

@@ -40,7 +40,7 @@ aside {
 #main {
   padding: 22px;
   margin-bottom: 35px;
-  background-color: #fcfcfa;
+  background-color: $main-bg;
   border: solid 1px #e2e0d8;
   @include border-radius(5px);
 

assets/sass/layout.scss

@@ -41,8 +41,7 @@ hgroup,
 main,
 menu,
 nav,
-section,
-summary {
+section {
   display: block;
 }
 

assets/sass/normalize.scss

@@ -50,12 +50,12 @@
     td.matches {
       padding: 0 !important;
       font-size: 12px;
-      font-weight: bold;
 
       a {
         display: block;
         padding: 5px 12px;
         margin-bottom: 0;
+        font-weight: bold;
         color: $font-color;
         transition-duration: 0s;
 
@@ -143,3 +143,43 @@ ol.full-search-results {
     margin-top: 4px;
   }
 }
+
+// Search results page
+#search-div {
+  border: solid 1px $base-border-color;
+  background: $main-bg;
+  padding: 22px;
+  @include border-radius(5px);
+
+  ol {
+    background-color: $callout-color;
+    @include border-radius(3px);
+    padding: 8px 14px 4px;
+    margin: 0 0 1.4em;
+    list-style: none;
+  }
+
+
+  .pagefind-ui__result-inner > .pagefind-ui__result-title {
+    font-size: 16px;
+    line-height: $base-line-height * 2;
+    font-weight: bold;
+  }
+
+  .pagefind-ui__result-tags {
+    list-style: none;
+    color: $light-font-color;
+    margin-bottom: 1.4em;
+    margin-left: 0;
+  }
+}
+
+button {
+  border-color: $link-color;
+  color: $link-color;
+  background: $callout-color;
+  font-weight: bold;
+  @include border-radius(3px);
+  outline: none;
+  padding: 0 0.5em;
+}

assets/sass/search.scss

@@ -32,3 +32,4 @@ $base-border-color:  #d8d7cf;
 $callout-color:      #e9e8e0;
 
 $black-3 : #333333;
+$main-bg: #fcfcfa;

assets/sass/variables.scss

@@ -0,0 +1,24 @@
+---
+section: search
+url: /search.html
+aliases:
+- /search/index.html
+---
+
+<header>Search is now client-side only!</header>
+
+<table>
+  <tbody><tr class="show-all">
+    <td class="category"> &nbsp; </td>
+    <td class="matches">
+      <ul>
+        <li>
+          <a class="highlight">Search is now client-side only!</a>
+        </li>
+        <li>
+          Since moving from a full Ruby on Rails app to a static site, the search on https://git-scm.com/ is implemented in Javascript, on the client side. Therefore, the server-side search via https://git-scm.com/search?search=&lt;search-term&gt; is no longer supported.
+        </li>
+      </ul>
+    </td>
+  </tr>
+</tbody></table>