Add tld_gen_phf.py to automate tlds.rs generation from tld_lib.yml.

The PHF hash set in tlds.rs was previously maintained by hand. This adds
a generator script (tld_gen_phf.py) and a wrapper (update_tlds.sh) to
produce it from the canonical tld_lib.yml source. Also adds 7 punycode
TLDs that were missing from the hand-maintained file and fixes sorting
of unicode TLDs containing non-ASCII Latin characters.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: sayrer

rust/conformance/BUILD.bazel

@@ -181,3 +181,23 @@ genrule(
     tools = ["tests/insert_tld_grammar.sh"],
     visibility = ["//visibility:public"],
 )
+
+# Hermetic PHF TLD hash set generator
+py_binary(
+    name = "tld_gen_phf",
+    srcs = ["tests/tld_gen_phf.py"],
+    main = "tests/tld_gen_phf.py",
+    python_version = "PY3",
+    deps = [
+        requirement("PyYAML"),
+    ],
+)
+
+genrule(
+    name = "generate_tlds_rs",
+    srcs = ["tests/tld_lib.yml"],
+    outs = ["tlds.rs"],
+    cmd = "$(location :tld_gen_phf) --input $(location tests/tld_lib.yml) --output $@",
+    tools = [":tld_gen_phf"],
+    visibility = ["//visibility:public"],
+)

rust/conformance/tests/tld_gen_phf.py

@@ -0,0 +1,169 @@
+# Copyright 2025 Robert Sayre
+# Licensed under the Apache License, Version 2.0
+# http://www.apache.org/licenses/LICENSE-2.0
+
+import argparse
+import sys
+
+import yaml
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate TLD PHF hash set for Rust"
+    )
+    parser.add_argument(
+        "--input", required=True, help="Input YAML file containing TLDs"
+    )
+    parser.add_argument(
+        "--output", required=True, help="Output file for generated tlds.rs"
+    )
+    args = parser.parse_args()
+
+    with open(args.input, "r", encoding="utf-8") as stream:
+        try:
+            tld_structure = yaml.safe_load(stream)
+        except yaml.YAMLError as exc:
+            print(exc, file=sys.stderr)
+            sys.exit(1)
+
+    tlds = set()
+    for kind in tld_structure.values():
+        for element in kind:
+            idna = element.encode("idna").decode("ascii")
+            if idna != element:
+                tlds.add(idna)
+            tlds.add(element.lower())
+
+    # Separate into ASCII, unicode, and punycode for organized output
+    ascii_tlds = sorted(t for t in tlds if all(ord(c) < 128 for c in t) and not t.startswith("xn--"))
+    unicode_tlds = sorted(t for t in tlds if any(ord(c) >= 128 for c in t))
+    punycode_tlds = sorted(t for t in tlds if t.startswith("xn--"))
+
+    with open(args.output, "w", encoding="utf-8") as out:
+        out.write(FILE_HEADER)
+
+        # Write phf_set entries
+        for tld in ascii_tlds:
+            out.write(f'    "{tld}",\n')
+        for tld in unicode_tlds:
+            out.write(f'    "{tld}",\n')
+        out.write("    // Punycode TLDs (IDN TLDs in ASCII-compatible encoding)\n")
+        for tld in punycode_tlds:
+            out.write(f'    "{tld}",\n')
+
+        out.write(FILE_FOOTER)
+
+    print(
+        f"Generated {args.output} with {len(tlds)} TLDs "
+        f"({len(ascii_tlds)} ASCII, {len(unicode_tlds)} Unicode, "
+        f"{len(punycode_tlds)} Punycode)",
+        file=sys.stderr,
+    )
+
+
+FILE_HEADER = """\
+// Copyright 2025 Robert Sayre
+// Licensed under the Apache License, Version 2.0
+// http://www.apache.org/licenses/LICENSE-2.0
+
+//! TLD lookup using a compile-time perfect hash map.
+//!
+//! This module provides O(1) TLD validation, replacing the O(n)
+//! Pest grammar alternation.
+//!
+//! This file is generated by tld_gen_phf.py from tld_lib.yml.
+//! Do not edit by hand.
+
+use phf::phf_set;
+
+/// A compile-time perfect hash set of all valid TLDs.
+/// Case-insensitive matching is handled by lowercasing input before lookup.
+pub static TLDS: phf::Set<&'static str> = phf_set! {
+"""
+
+FILE_FOOTER = """\
+};
+
+/// Check if a string is a valid TLD.
+/// The input should be lowercase.
+#[inline]
+pub fn is_valid_tld(tld: &str) -> bool {
+    TLDS.contains(tld)
+}
+
+/// Maximum TLD length in bytes (longest is "சிங்கப்பூர்" at 33 bytes)
+const MAX_TLD_LEN: usize = 64;
+
+/// Check if a string is a valid TLD, case-insensitively.
+/// Uses a stack-allocated buffer for lowercase conversion to avoid heap allocations.
+#[inline]
+pub fn is_valid_tld_case_insensitive(tld: &str) -> bool {
+    // Fast path: if already lowercase ASCII, check directly
+    if tld.bytes().all(|b| !b.is_ascii_uppercase()) {
+        return TLDS.contains(tld);
+    }
+
+    // Need to lowercase - use stack buffer if small enough
+    if tld.len() <= MAX_TLD_LEN {
+        let mut buf = [0u8; MAX_TLD_LEN];
+        let mut i = 0;
+        for c in tld.chars() {
+            for lc in c.to_lowercase() {
+                let len = lc.len_utf8();
+                if i + len > MAX_TLD_LEN {
+                    // Overflow - fall back to heap allocation
+                    return TLDS.contains(&tld.to_lowercase());
+                }
+                lc.encode_utf8(&mut buf[i..]);
+                i += len;
+            }
+        }
+        // Safety: we only wrote valid UTF-8 chars
+        let lowered = unsafe { std::str::from_utf8_unchecked(&buf[..i]) };
+        TLDS.contains(lowered)
+    } else {
+        // TLD too long - fall back to heap allocation (shouldn't happen with valid TLDs)
+        TLDS.contains(&tld.to_lowercase())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_common_tlds() {
+        assert!(is_valid_tld("com"));
+        assert!(is_valid_tld("org"));
+        assert!(is_valid_tld("net"));
+        assert!(is_valid_tld("uk"));
+        assert!(is_valid_tld("io"));
+    }
+
+    #[test]
+    fn test_unicode_tlds() {
+        assert!(is_valid_tld("中国"));
+        assert!(is_valid_tld("рф"));
+        assert!(is_valid_tld("한국"));
+    }
+
+    #[test]
+    fn test_punycode_tlds() {
+        assert!(is_valid_tld("xn--p1ai")); // .рф (Russia)
+        assert!(is_valid_tld("xn--wgbh1c")); // .مصر (Egypt)
+        assert!(is_valid_tld("xn--mgbaam7a8h")); // .امارات (UAE)
+    }
+
+    #[test]
+    fn test_invalid_tlds() {
+        assert!(!is_valid_tld("invalid"));
+        assert!(!is_valid_tld("notarealtld"));
+        assert!(!is_valid_tld(""));
+    }
+}
+"""
+
+
+if __name__ == "__main__":
+    main()

rust/twitter-text/src/tlds.rs

@@ -6,6 +6,9 @@
 //!
 //! This module provides O(1) TLD validation, replacing the O(n)
 //! Pest grammar alternation.
+//!
+//! This file is generated by tld_gen_phf.py from tld_lib.yml.
+//! Do not edit by hand.
 
 use phf::phf_set;
 
@@ -1340,8 +1343,6 @@ pub static TLDS: phf::Set<&'static str> = phf_set! {
     "vegas",
     "ventures",
     "verisign",
-    "vermögensberater",
-    "vermögensberatung",
     "versicherung",
     "vet",
     "vg",
@@ -1437,6 +1438,8 @@ pub static TLDS: phf::Set<&'static str> = phf_set! {
     "zone",
     "zuerich",
     "zw",
+    "vermögensberater",
+    "vermögensberatung",
     "ελ",
     "ευ",
     "бг",
@@ -1603,6 +1606,7 @@ pub static TLDS: phf::Set<&'static str> = phf_set! {
     "xn--3ds443g",
     "xn--3e0b707e",
     "xn--3hcrj9c",
+    "xn--3oq18vl8pn36a",
     "xn--3pxu8k",
     "xn--42c2d9a",
     "xn--45br5cyl",
@@ -1645,6 +1649,7 @@ pub static TLDS: phf::Set<&'static str> = phf_set! {
     "xn--e1a4c",
     "xn--eckvdtc9d",
     "xn--efvy88h",
+    "xn--estv75g",
     "xn--fct429k",
     "xn--fhbei",
     "xn--fiq228c5hs",
@@ -1671,22 +1676,26 @@ pub static TLDS: phf::Set<&'static str> = phf_set! {
     "xn--j1amh",
     "xn--j6w193g",
     "xn--jlq480n2rg",
+    "xn--jlq61u9w7b",
     "xn--jvr189m",
     "xn--kcrx77d1x4a",
     "xn--kprw13d",
     "xn--kpry57d",
+    "xn--kpu716f",
     "xn--kput3i",
     "xn--l1acc",
     "xn--lgbbat1ad8j",
     "xn--mgb9awbf",
     "xn--mgba3a3ejt",
     "xn--mgba3a4f16a",
     "xn--mgba7c0bbn0a",
+    "xn--mgbaakc7dvf",
     "xn--mgbaam7a8h",
     "xn--mgbab2bd",
     "xn--mgbah1a3hjkrd",
     "xn--mgbai9azgqp6j",
     "xn--mgbayh7gpa",
+    "xn--mgbb9fbpob",
     "xn--mgbbh1a",
     "xn--mgbbh1a71e",
     "xn--mgbc0a9azcg",
@@ -1714,6 +1723,7 @@ pub static TLDS: phf::Set<&'static str> = phf_set! {
     "xn--otu796d",
     "xn--p1acf",
     "xn--p1ai",
+    "xn--pbt977c",
     "xn--pgbs0dh",
     "xn--pssy2u",
     "xn--q7ce6a",

update_tlds.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+# Copyright 2025 Robert Sayre
+# Licensed under the Apache License, Version 2.0
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Regenerate TLD files from tld_lib.yml.
+# Requires: Python 3, PyYAML
+#
+# By default, only regenerates tlds.rs (the PHF hash set used at runtime).
+# The Pest grammar uses character-class-based TLD matching and does not
+# need regeneration when TLDs change.
+#
+# To also regenerate the Pest trie grammar (legacy, requires patricia-trie):
+#   ./update_tlds.sh --pest
+#
+# Usage: ./update_tlds.sh [--pest]
+
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "$0")" && pwd)"
+
+TLD_YAML="$REPO_ROOT/rust/conformance/tests/tld_lib.yml"
+TLD_GEN_PHF="$REPO_ROOT/rust/conformance/tests/tld_gen_phf.py"
+TLDS_RS="$REPO_ROOT/rust/twitter-text/src/tlds.rs"
+
+UPDATE_PEST=false
+for arg in "$@"; do
+    case "$arg" in
+        --pest) UPDATE_PEST=true ;;
+        *) echo "Unknown option: $arg" >&2; exit 1 ;;
+    esac
+done
+
+if [ "$UPDATE_PEST" = true ]; then
+    TLD_GEN="$REPO_ROOT/rust/conformance/tests/tld_gen.py"
+    INSERT_SCRIPT="$REPO_ROOT/rust/conformance/tests/insert_tld_grammar.sh"
+    PEST_FILE="$REPO_ROOT/rust/parser/src/twitter_text.pest"
+
+    TMPDIR="$(mktemp -d)"
+    trap 'rm -rf "$TMPDIR"' EXIT
+
+    echo "==> Generating Pest TLD grammar..."
+    python3 "$TLD_GEN" \
+        --input "$TLD_YAML" \
+        --output "$TMPDIR/generated_tld.pest"
+
+    echo "==> Inserting TLD grammar into twitter_text.pest..."
+    bash "$INSERT_SCRIPT" \
+        "$PEST_FILE" \
+        "$TMPDIR/generated_tld.pest" \
+        "$PEST_FILE"
+fi
+
+echo "==> Generating tlds.rs PHF hash set..."
+python3 "$TLD_GEN_PHF" \
+    --input "$TLD_YAML" \
+    --output "$TLDS_RS"
+
+echo ""
+echo "Done. Updated:"
+if [ "$UPDATE_PEST" = true ]; then
+    echo "  - $PEST_FILE"
+fi
+echo "  - $TLDS_RS"