feat(fe): support Unicode 16 in identifiers

Unicode 16 added some new characters which are legal in identifiers, such as
U+1C89 CYRILLIC CAPITAL LETTER TJE. Update our tables for Unicode 16.

Author: strager

docs/CHANGELOG.md

@@ -17,6 +17,8 @@ Semantic Versioning.
   string and so will never equal undefined; use 'undefined' instead").
   (Implemented by [CoderMuffin][].) ([#1207][])
 * Emacs: The Eglot plugin now supports `typescript-mode`.
+* Unicode 16 is now supported, including new Cyrillic code points in
+  identifiers.
 
 ### Fixed
 

src/quick-lint-js/fe/lex-unicode-generated.cpp

@@ -24,8 +24,8 @@ std::string pretty(char32_t c) {
 }
 
 bool icu_data_is_valid() {
-  std::uint8_t minimum_unicode_major_version = 15;
-  std::uint8_t minimum_unicode_minor_version = 1;
+  std::uint8_t minimum_unicode_major_version = 16;
+  std::uint8_t minimum_unicode_minor_version = 0;
 
   UVersionInfo version;
   ::u_getUnicodeVersion(version);

test/test-lex-unicode.cpp

@@ -1473,6 +1473,9 @@ TEST_F(Test_Lex, non_ascii_identifier) {
 
   // KHOJKI LETTER QA, introduced in Unicode 15.
   this->check_single_token(u8"\U0001123f"_sv, u8"\U0001123f"_sv);
+
+  // CYRILLIC CAPITAL LETTER TJE, introduced in Unicode 16.
+  this->check_single_token(u8"\u1c89"_sv, u8"\u1c89"_sv);
 }
 
 TEST_F(Test_Lex, non_ascii_identifier_with_escape_sequence) {

test/test-lex.cpp

@@ -169,10 +169,10 @@ function isIDContinue(codePoint) {
 }
 
 let ID_START_CODE_POINTS = new Set(
-  require("@unicode/unicode-15.1.0/Binary_Property/ID_Start/code-points.js")
+  require("@unicode/unicode-16.0.0/Binary_Property/ID_Start/code-points.js")
 );
 let ID_CONTINUE_CODE_POINTS = new Set(
-  require("@unicode/unicode-15.1.0/Binary_Property/ID_Continue/code-points.js")
+  require("@unicode/unicode-16.0.0/Binary_Property/ID_Continue/code-points.js")
 );
 
 // Pattern_White_Space

tools/generate-lex-unicode/index.js

@@ -3,8 +3,7 @@
     "fmt": "prettier --write '*.js' '*.json'"
   },
   "dependencies": {
-    "@unicode/unicode-15.0.0": "*",
-    "@unicode/unicode-15.1.0": "*"
+    "@unicode/unicode-16.0.0": "*"
   },
   "devDependencies": {
     "prettier": "^2.8.4"

tools/generate-lex-unicode/package.json

@@ -2,15 +2,10 @@
 # yarn lockfile v1
 
 
-"@unicode/unicode-15.0.0@*":
-  version "1.5.2"
-  resolved "https://registry.yarnpkg.com/@unicode/unicode-15.0.0/-/unicode-15.0.0-1.5.2.tgz#5350ad022050ca5e165f8352d9f971d418c99deb"
-  integrity sha512-PepMvMxf9j4sp4bZn7W9JJoMxynHk66ZPCsx6n3v47T5vmM+qfIy0z1MMU+EDmRZr2cvs1aT9ZwUEMRPVXR23g==
-
-"@unicode/unicode-15.1.0@*":
-  version "1.5.2"
-  resolved "https://registry.yarnpkg.com/@unicode/unicode-15.1.0/-/unicode-15.1.0-1.5.2.tgz#0358d05ab99a7d05c12c68cc2123cf124c34beb9"
-  integrity sha512-7PAgnShDr8ziK6XeHB/TUVFboDFEhaQKKyrw55/Kx9o6AQDy1s7dJ9KRpRerW9nrR5qMGUQvOqTXOAek6ZIXkg==
+"@unicode/unicode-16.0.0@*":
+  version "1.6.6"
+  resolved "https://registry.yarnpkg.com/@unicode/unicode-16.0.0/-/unicode-16.0.0-1.6.6.tgz#b4c4b7d2b8976cb6b322ff553308862e6da0cc23"
+  integrity sha512-NQRvvfCYA+bZztjMRN8xgAELYDxBRDHEQaU648UYd4leEUkiEOBlezGLvZKvzA9wiN04GbNypXcrEGQxuU0Wxw==
 
 prettier@^2.8.4:
   version "2.8.4"