html: implement language code validation (#93)

* html: lang: implement language code validation

* html: lang: add completion for language and script

* html: lang: implement iso 639 and iso 15924 check

* html: lang: parse iana registry

registry: https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry

* html: lang: add check for grandfathered tags

* html: lang: rewrite parser and use static maps for lookup

* html: lang: add completion for region

* html: lang: implement rfc 5646

* html: lang: add registry fetch step to build.zig

* html: lang: propose region completion after any number of dashes

it can be prepended by any number of extlang tags and a script tag

* html: lang: check extlang and variant prefix

* html: lang: use rule for global lang attribute

* html: lang: move validate and completions to module

* html: lang: fix multi-subtag prefixes

* html: lang: support subtags with multiple prefixes

* html: lang: add test for deprecation

* fix import

---------

Co-authored-by: Loris Cro <kappaloris@gmail.com>

Author: andreafeletto

build.zig

@@ -26,6 +26,18 @@ pub fn build(b: *std.Build) !void {
     superhtml.addImport("scripty", scripty.module("scripty"));
     superhtml.addImport("tracy", tracy.module("tracy"));
 
+    const language_tag_parser = b.addExecutable(.{
+        .name = "language-tag-parser",
+        .root_module = b.createModule(.{
+            .root_source_file = b.path("src/html/language_tag/parse.zig"),
+            .target = target,
+        }),
+    });
+    const language_tag_parse = b.addRunArtifact(language_tag_parser);
+    superhtml.addImport("language-tag-registry", b.createModule(.{
+        .root_source_file = language_tag_parse.addOutputFileArg("registry.zon"),
+    }));
+
     if (target.result.os.tag == .windows) {
         superhtml.linkSystemLibrary("advapi32", .{});
 
@@ -61,6 +73,7 @@ pub fn build(b: *std.Build) !void {
     setupTestStep(b, superhtml, check);
     setupCliTool(b, target, optimize, options, superhtml, folders, lsp);
     setupWasmStep(b, optimize, options, superhtml, lsp);
+    setupFetchLanguageSubtagRegistryStep(b, target);
     if (version == .tag) {
         setupReleaseStep(b, options, superhtml, folders, lsp);
     }
@@ -181,6 +194,27 @@ fn setupWasmStep(
     wasm.dependOn(&target_output.step);
 }
 
+fn setupFetchLanguageSubtagRegistryStep(
+    b: *std.Build,
+    target: std.Build.ResolvedTarget,
+) void {
+    const step = b.step(
+        "fetch-language-subtag-registry",
+        "Fetch the IANA language subtag registry",
+    );
+    const fetcher = b.addExecutable(.{
+        .name = "language-subtag-fetcher",
+        .root_module = b.createModule(.{
+            .root_source_file = b.path("src/html/language_tag/fetch.zig"),
+            .target = target,
+        }),
+    });
+    const fetch = b.addRunArtifact(fetcher);
+    fetch.has_side_effects = true;
+    fetch.addFileArg(b.path("src/html/language_tag/registry.txt"));
+    step.dependOn(&fetch.step);
+}
+
 fn setupReleaseStep(
     b: *std.Build,
     options: *std.Build.Step.Options,

src/cli/lsp.zig

@@ -125,7 +125,7 @@ pub fn initialize(
             .triggerCharacters = &.{
                 "<",  "/", " ",
                 "\n", "'", "\"",
-                "=",  ",",
+                "=",  ",", "-",
             },
         },
 

src/html/Attribute.zig

@@ -9,6 +9,7 @@ const root = @import("../root.zig");
 const Language = root.Language;
 const Span = root.Span;
 const log = std.log.scoped(.attribute);
+const language_tag = @import("language_tag.zig");
 
 rule: Rule,
 desc: []const u8,
@@ -74,6 +75,9 @@ pub const Rule = union(enum) {
     /// MIME
     mime,
 
+    /// BCP 47 language tag
+    lang,
+
     /// Not negative integer
     non_neg_int: struct {
         min: usize = 0,
@@ -257,6 +261,20 @@ pub const Rule = union(enum) {
             },
             .mime => return validateMime(gpa, errors, src, node_idx, attr),
             .cors => continue :rule .{ .list = cors_list },
+            .lang => {
+                const value = attr.value orelse return;
+                const value_slice = value.span.slice(src);
+                if (language_tag.validate(value_slice)) |rejection| return errors.append(gpa, .{
+                    .tag = .{
+                        .invalid_attr_value = .{ .reason = rejection.reason },
+                    },
+                    .main_location = .{
+                        .start = value.span.start + rejection.offset,
+                        .end = value.span.start + rejection.offset + rejection.length,
+                    },
+                    .node_idx = node_idx,
+                });
+            },
             .not_empty => {
                 const value = attr.value orelse return errors.append(gpa, .{
                     .tag = .missing_attr_value,
@@ -944,6 +962,9 @@ pub fn completions(
                     .cors => if (value_content.len == 0) {
                         return Rule.cors_list.completions;
                     },
+                    .lang => {
+                        return language_tag.completions(value_content);
+                    },
                     .list => |l| {
                         if (value_content.len == 0) {
                             return l.completions;
@@ -1625,7 +1646,7 @@ pub const global: AttributeSet = .init(&.{
     .{
         .name = "lang",
         .model = .{
-            .rule = .not_empty,
+            .rule = .lang,
             .desc = "The `lang` global attribute helps define the language of an element: the language that non-editable elements are written in, or the language that the editable elements should be written in by the user. The attribute contains a single BCP 47 language tag.",
         },
     },

src/html/elements/a.zig

@@ -98,7 +98,7 @@ pub const attributes: AttributeSet = .init(&.{
     .{
         .name = "hreflang",
         .model = .{
-            .rule = .any,
+            .rule = .lang,
             .desc =
             \\Hints at the human language of the linked URL. No built-in
             \\functionality. Allowed values are the same as the global lang

src/html/elements/link.zig

@@ -187,7 +187,7 @@ pub const attributes: AttributeSet = .init(&.{
     .{
         .name = "hreflang",
         .model = .{
-            .rule = .not_empty, // TODO validate
+            .rule = .lang,
             .desc = "This attribute indicates the language of the linked resource. It is purely advisory.",
         },
     },

src/html/elements/track.zig

@@ -79,7 +79,7 @@ pub const attributes: AttributeSet = .init(&.{
     .{
         .name = "srclang",
         .model = .{
-            .rule = .not_empty,
+            .rule = .lang,
             .desc = "Language of the track text data. It must be a valid BCP 47 language tag. If the `kind` attribute is set to 'subtitles', then `srclang` must be defined.",
         },
     },

src/html/language_tag.zig

@@ -0,0 +1,216 @@
+const std = @import("std");
+const Completion = @import("Ast.zig").Completion;
+const Registry = @import("language_tag/parse.zig").Registry;
+const registry: Registry = @import("language-tag-registry");
+
+pub const Rejection = struct {
+    reason: []const u8,
+    offset: u32,
+    length: u32,
+
+    pub fn init(bytes: []const u8, subtag: []const u8, reason: []const u8) Rejection {
+        return .{
+            .reason = reason,
+            .offset = @intCast(@intFromPtr(subtag.ptr) - @intFromPtr(bytes.ptr)),
+            .length = @intCast(subtag.len),
+        };
+    }
+};
+
+pub fn validate(bytes: []const u8) ?Rejection {
+    if (maps.grandfathered.get(bytes)) |data| {
+        if (!data.is_deprecated) return null;
+    }
+
+    const ParseState = enum {
+        language,
+        extlang,
+        script,
+        region,
+        variant,
+        singleton,
+        extension,
+        extension_extra,
+        privateuse,
+        privateuse_extra,
+    };
+    var parse_state: ParseState = .language;
+
+    var subtags = std.mem.splitScalar(u8, bytes, '-');
+    while (subtags.next()) |subtag| state: switch (parse_state) {
+        .language => switch (subtag.len) {
+            0 => return .init(bytes, subtag, "cannot be empty"),
+            1 => return .init(bytes, subtag, "too short"),
+            2...8 => {
+                if (maps.language.get(subtag)) |data| {
+                    if (data.is_deprecated) return .init(bytes, subtag, "deprecated language");
+                } else {
+                    return .init(bytes, subtag, "unknown language");
+                }
+                parse_state = .extlang;
+            },
+            else => return .init(bytes, subtag, "too long"),
+        },
+        .extlang => switch (subtag.len) {
+            3 => {
+                if (std.ascii.isDigit(subtag[0])) continue :state .region;
+                if (maps.extlang.get(subtag)) |data| {
+                    if (data.is_deprecated) return .init(bytes, subtag, "deprecated language extension");
+                    for (data.prefixes) |prefix| {
+                        if (std.ascii.startsWithIgnoreCase(bytes, prefix)) break;
+                    } else {
+                        return .init(bytes, subtag, "incompatible language extension");
+                    }
+                } else {
+                    return .init(bytes, subtag, "unknown language extension");
+                }
+                parse_state = .script;
+            },
+            else => continue :state .script,
+        },
+        .script => switch (subtag.len) {
+            4 => {
+                if (std.ascii.isDigit(subtag[0])) continue :state .variant;
+                if (!maps.script.has(subtag)) {
+                    return .init(bytes, subtag, "unknown language script");
+                }
+                parse_state = .region;
+            },
+            else => continue :state .region,
+        },
+        .region => switch (subtag.len) {
+            2...3 => {
+                if (maps.region.get(subtag)) |data| {
+                    if (data.is_deprecated) return .init(bytes, subtag, "deprecated language region");
+                } else {
+                    return .init(bytes, subtag, "unknown language region");
+                }
+                parse_state = .variant;
+            },
+            else => continue :state .variant,
+        },
+        .variant => switch (subtag.len) {
+            4...8 => {
+                if (maps.variant.get(subtag)) |data| {
+                    if (data.is_deprecated) return .init(bytes, subtag, "deprecated language variant");
+                    for (data.prefixes) |prefix| {
+                        if (std.ascii.startsWithIgnoreCase(bytes, prefix)) break;
+                    } else {
+                        return .init(bytes, subtag, "incompatible language variant");
+                    }
+                } else {
+                    return .init(bytes, subtag, "unknown language variant");
+                }
+                parse_state = .variant;
+            },
+            else => continue :state .singleton,
+        },
+        .singleton => {
+            if (subtag.len != 1) {
+                return .init(bytes, subtag, "extension prefix must be a single character");
+            }
+            parse_state = switch (std.ascii.toLower(subtag[0])) {
+                'x' => .privateuse,
+                'a'...'w', 'y'...'z', '0'...'9' => .extension,
+                else => return .init(bytes, subtag, "extension prefix must be alphanumeric"),
+            };
+        },
+        .extension => switch (subtag.len) {
+            2...8 => {
+                for (subtag) |char| if (!std.ascii.isAlphanumeric(char)) {
+                    return .init(bytes, subtag, "extension must be alphanumeric");
+                };
+                parse_state = .extension_extra;
+            },
+            else => return .init(bytes, subtag, "wrong extension length"),
+        },
+        .extension_extra => switch (subtag.len) {
+            2...8 => continue :state .extension,
+            else => continue :state .singleton,
+        },
+        .privateuse => switch (subtag.len) {
+            1...8 => {
+                for (subtag) |char| if (!std.ascii.isAlphanumeric(char)) {
+                    return .init(bytes, subtag, "private use extension must be alphanumeric");
+                };
+                parse_state = .privateuse_extra;
+            },
+            else => return .init(bytes, subtag, "wrong private use extension length"),
+        },
+        .privateuse_extra => switch (subtag.len) {
+            1...8 => continue :state .privateuse,
+            else => return .init(bytes, subtag, "subtag after private use extension"),
+        },
+    };
+    return null;
+}
+
+pub fn completions(value: []const u8) []const Completion {
+    if (value.len == 0) {
+        return &language_completions;
+    }
+
+    if (std.mem.endsWith(u8, value, "-")) {
+        return &region_completions;
+    }
+
+    return &.{};
+}
+
+const Map = std.StaticStringMapWithEql(Registry.Subtag.Data, std.ascii.eqlIgnoreCase);
+
+const maps = struct {
+    pub const language = makeMap("language");
+    pub const extlang = makeMap("extlang");
+    pub const script = makeMap("script");
+    pub const region = makeMap("region");
+    pub const variant = makeMap("variant");
+    pub const grandfathered = makeMap("grandfathered");
+};
+
+fn makeMap(comptime kind: []const u8) Map {
+    const KV = struct { []const u8, Registry.Subtag.Data };
+    const subtags = @field(registry, kind);
+    @setEvalBranchQuota(subtags.len * 2);
+    var kvs: [subtags.len]KV = undefined;
+    for (subtags, &kvs) |subtag, *kv| {
+        kv.* = .{ subtag.name, subtag.data };
+    }
+    return .initComptime(kvs);
+}
+
+const language_completions = makeCompletions("language");
+const region_completions = makeCompletions("region");
+
+fn makeCompletions(comptime kind: []const u8) [@field(registry, kind).len]Completion {
+    const subtags = @field(registry, kind);
+    @setEvalBranchQuota(subtags.len * 2);
+    var comps: [subtags.len]Completion = undefined;
+    for (subtags, &comps) |subtag, *comp| {
+        comp.* = .{
+            .label = subtag.name,
+            .desc = subtag.data.description orelse subtag.name,
+        };
+    }
+    return comps;
+}
+
+test "validate: all subtags" {
+    const value = "sgn-ase-Latn-US-blasl-a-abcd-x-1234";
+    try std.testing.expectEqual(null, validate(value));
+}
+
+test "validate: deprecated language" {
+    const value = "in";
+    try std.testing.expect(validate(value) != null);
+}
+
+test "validate: multiple prefixes" {
+    const valid_1 = "sgn-ase-blasl";
+    const valid_2 = "ase-blasl";
+    try std.testing.expectEqual(null, validate(valid_1));
+    try std.testing.expectEqual(null, validate(valid_2));
+
+    const invalid = "it-blasl";
+    try std.testing.expect(validate(invalid) != null);
+}

src/html/language_tag/fetch.zig

@@ -0,0 +1,26 @@
+const std = @import("std");
+
+pub fn main() !void {
+    var arena_instance: std.heap.ArenaAllocator = .init(std.heap.page_allocator);
+    const arena = arena_instance.allocator();
+    defer arena_instance.deinit();
+
+    var args = try std.process.argsWithAllocator(arena);
+    std.debug.assert(args.skip());
+
+    const file = file: {
+        const output_path = args.next() orelse break :file std.fs.File.stdout();
+        break :file try std.fs.cwd().createFile(output_path, .{});
+    };
+    var buffer: [1024]u8 = undefined;
+    var writer = file.writer(&buffer);
+    const output = &writer.interface;
+
+    var client: std.http.Client = .{ .allocator = arena };
+    _ = try client.fetch(.{
+        .location = .{ .url = "https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry" },
+        .response_writer = output,
+    });
+
+    try output.flush();
+}