Generate src/html/entities.cr automatically (#13998)

Author: HertzDevil

.gitattributes

@@ -10,6 +10,8 @@ lib/** linguist-vendored
 
 # produced by scripts/generate_windows_zone_names.cr
 src/crystal/system/win32/zone_names.cr linguist-generated
+# produced by scripts/generate_html_entities.cr
+src/html/entities.cr linguist-generated
 # produced by scripts/generate_ssl_server_defaults.cr
 src/openssl/ssl/defaults.cr linguist-generated
 # produced by scripts/generate_grapheme_properties.cr

Makefile

@@ -161,6 +161,10 @@ generate_data: src/crystal/system/win32/zone_names.cr
 src/crystal/system/win32/zone_names.cr: scripts/generate_windows_zone_names.cr
 	$(CRYSTAL) run $<
 
+generate_data: src/html/entities.cr
+src/html/entities.cr: scripts/generate_html_entities.cr scripts/html_entities.ecr
+	$(CRYSTAL) run $<
+
 .PHONY: install
 install: $(O)/crystal man/crystal.1.gz ## Install the compiler at DESTDIR
 	$(INSTALL) -d -m 0755 "$(BINDIR)/"

Makefile.win

@@ -163,6 +163,10 @@ generate_data: src\crystal\system\win32\zone_names.cr
 src\crystal\system\win32\zone_names.cr: scripts\generate_windows_zone_names.cr
 	$(CRYSTAL) run $<
 
+generate_data: src\html\entities.cr
+src\html\entities.cr: scripts\generate_html_entities.cr scripts\html_entities.ecr
+	$(CRYSTAL) run $<
+
 .PHONY: install
 install: $(O)\crystal.exe ## Install the compiler at prefix
 	$(call MKDIR,"$(BINDIR)")

scripts/generate_html_entities.cr

@@ -0,0 +1,43 @@
+#! /usr/bin/env crystal
+
+require "http"
+require "json"
+require "ecr"
+
+record Entity, characters : String, codepoints : Array(Int32) do
+  include JSON::Serializable
+  include JSON::Serializable::Strict
+end
+
+single_char_entities = [] of {String, Entity}
+double_char_entities = [] of {String, Entity}
+
+HTTP::Client.get("https://html.spec.whatwg.org/entities.json") do |res|
+  Hash(String, Entity).from_json(res.body_io).each do |name, entity|
+    name = name.rchop(';').lchop?('&') || raise "Entity does not begin with &"
+
+    entities =
+      case entity.codepoints.size
+      when 1; single_char_entities
+      when 2; double_char_entities
+      else    raise "Unknown entity codepoint size"
+      end
+
+    entities << {name, entity}
+  end
+end
+
+single_char_entities.uniq!(&.first).sort_by!(&.first)
+double_char_entities.uniq!(&.first).sort_by!(&.first)
+
+max_entity_name_size = {
+  single_char_entities.max_of { |name, _| name.size },
+  double_char_entities.max_of { |name, _| name.size },
+}.max
+
+path = "#{__DIR__}/../src/html/entities.cr"
+File.open(path, "w") do |file|
+  ECR.embed "#{__DIR__}/html_entities.ecr", file
+end
+
+`crystal tool format #{path}`

scripts/html_entities.ecr

@@ -0,0 +1,24 @@
+# This file was automatically generated by running:
+#
+#   scripts/generate_html_entities.cr
+#
+# DO NOT EDIT
+
+module HTML
+  # :nodoc:
+  SINGLE_CHAR_ENTITIES = {
+    <%- single_char_entities.each do |name, entity| -%>
+      <%= name.inspect %>.to_slice => '\u{<%= "%06X" % entity.codepoints[0] %>}',
+    <%- end -%>
+  } of Bytes => Char
+
+  # :nodoc:
+  DOUBLE_CHAR_ENTITIES = {
+    <%- double_char_entities.each do |name, entity| -%>
+      <%= name.inspect %>.to_slice => "\u{<%= "%04X" % entity.codepoints[0] %>}\u{<%= "%04X" % entity.codepoints[1] %>}",
+    <%- end -%>
+  } of Bytes => String
+
+  # :nodoc:
+  MAX_ENTITY_NAME_SIZE = <%= max_entity_name_size %>
+end