Merge pull request #3992 from ruby/claude/cloudflare-error-pr-3989-a9b7cf

Read the Pagefind entry as UTF-8

Author: hsbt

lib/search_index.rb

@@ -86,8 +86,13 @@ def index_key(lang)
     lang.tr("_", "-").downcase
   end
 
+  # The entry is UTF-8 no matter what locale the build runs under: Pagefind
+  # lists the word characters it splits on in `include_characters`, and those
+  # reach past ASCII. Build hosts that leave the locale at POSIX, Cloudflare
+  # Pages among them, give `File.read` a US-ASCII string, and JSON.parse raises
+  # on the first of those bytes as it converts the source to UTF-8.
   def read_entry
-    JSON.parse(File.read(@entry_path))
+    JSON.parse(File.read(@entry_path, encoding: Encoding::UTF_8))
   rescue Errno::ENOENT
     raise Error, "#{@entry_path} is missing, Pagefind wrote no index"
   rescue JSON::ParserError => e

test/test_search_index.rb

@@ -17,12 +17,31 @@
     teardown_tempdir
   end
 
+  # `include_characters` is the list of characters Pagefind counts as part of a
+  # word. It reaches past ASCII, which is what makes the entry a UTF-8 file.
   def write_entry(languages)
-    File.write(@entry_path, JSON.generate({ "version" => "1.5.2", "languages" => languages }))
+    entry = {
+      "version" => "1.5.2",
+      "languages" => languages,
+      "include_characters" => ["_", "‿", "⁀", "＿"],
+    }
+    File.write(@entry_path, JSON.generate(entry))
   end
 
   def read_entry
-    JSON.parse(File.read(@entry_path))
+    JSON.parse(File.read(@entry_path, encoding: Encoding::UTF_8))
+  end
+
+  # Stands in for a build host that leaves the locale at POSIX, which is what
+  # Cloudflare Pages does.
+  def with_default_external(encoding)
+    original = Encoding.default_external
+    verbose, $VERBOSE = $VERBOSE, nil
+    Encoding.default_external = encoding
+    yield
+  ensure
+    Encoding.default_external = original
+    $VERBOSE = verbose
   end
 
   def search_index(pagefind: SearchIndex::PAGEFIND)
@@ -148,6 +167,14 @@ def stub_pagefind(status)
       _(error.message).must_match(/missing index "en"/)
     end
 
+    it "reads the entry as UTF-8 whatever locale the build runs under" do
+      write_entry({ "en" => { "hash" => "en_abc", "wasm" => "en", "page_count" => 560 } })
+
+      with_default_external(Encoding::US_ASCII) { search_index.apply_fallbacks }
+
+      _(read_entry["languages"]["bg"]["hash"]).must_equal "en_abc"
+    end
+
     it "raises when the bundle has no languages" do
       write_entry({})