unicode-table-generator refactors

library/core/src/unicode/unicode_data.rs

@@ -1,4 +1,16 @@
 ///! This file is generated by `./x run src/tools/unicode-table-generator`; do not edit manually!
+// Alphabetic      :  1727 bytes, 142759 codepoints in 757 ranges (U+000041 - U+0323B0) using skiplist
+// Case_Ignorable  :  1053 bytes,   2749 codepoints in 452 ranges (U+000027 - U+0E01F0) using skiplist
+// Cased           :   407 bytes,   4578 codepoints in 159 ranges (U+000041 - U+01F18A) using skiplist
+// Cc              :     9 bytes,     65 codepoints in   2 ranges (U+000000 - U+0000A0) using skiplist
+// Grapheme_Extend :   887 bytes,   2193 codepoints in 375 ranges (U+000300 - U+0E01F0) using skiplist
+// Lowercase       :   935 bytes,   2569 codepoints in 675 ranges (U+000061 - U+01E944) using bitset
+// N               :   457 bytes,   1911 codepoints in 144 ranges (U+000030 - U+01FBFA) using skiplist
+// Uppercase       :   799 bytes,   1978 codepoints in 656 ranges (U+000041 - U+01F18A) using bitset
+// White_Space     :   256 bytes,     25 codepoints in  10 ranges (U+000009 - U+003001) using cascading
+// to_lower        : 11484 bytes
+// to_upper        : 13432 bytes
+// Total           : 31446 bytes
 
 #[inline(always)]
 const fn bitset_search<
@@ -772,7 +784,7 @@ pub mod conversions {
         }
     }
 
-    static LOWERCASE_TABLE: &[(char, u32)] = &[
+    static LOWERCASE_TABLE: &[(char, u32); 1434] = &[
         ('\u{c0}', 224), ('\u{c1}', 225), ('\u{c2}', 226), ('\u{c3}', 227), ('\u{c4}', 228),
         ('\u{c5}', 229), ('\u{c6}', 230), ('\u{c7}', 231), ('\u{c8}', 232), ('\u{c9}', 233),
         ('\u{ca}', 234), ('\u{cb}', 235), ('\u{cc}', 236), ('\u{cd}', 237), ('\u{ce}', 238),
@@ -1122,11 +1134,11 @@ pub mod conversions {
         ('\u{1e921}', 125251),
     ];
 
-    static LOWERCASE_TABLE_MULTI: &[[char; 3]] = &[
+    static LOWERCASE_TABLE_MULTI: &[[char; 3]; 1] = &[
         ['i', '\u{307}', '\u{0}'],
     ];
 
-    static UPPERCASE_TABLE: &[(char, u32)] = &[
+    static UPPERCASE_TABLE: &[(char, u32); 1526] = &[
         ('\u{b5}', 924), ('\u{df}', 4194304), ('\u{e0}', 192), ('\u{e1}', 193), ('\u{e2}', 194),
         ('\u{e3}', 195), ('\u{e4}', 196), ('\u{e5}', 197), ('\u{e6}', 198), ('\u{e7}', 199),
         ('\u{e8}', 200), ('\u{e9}', 201), ('\u{ea}', 202), ('\u{eb}', 203), ('\u{ec}', 204),
@@ -1499,7 +1511,7 @@ pub mod conversions {
         ('\u{1e941}', 125215), ('\u{1e942}', 125216), ('\u{1e943}', 125217),
     ];
 
-    static UPPERCASE_TABLE_MULTI: &[[char; 3]] = &[
+    static UPPERCASE_TABLE_MULTI: &[[char; 3]; 102] = &[
         ['S', 'S', '\u{0}'], ['\u{2bc}', 'N', '\u{0}'], ['J', '\u{30c}', '\u{0}'],
         ['\u{399}', '\u{308}', '\u{301}'], ['\u{3a5}', '\u{308}', '\u{301}'],
         ['\u{535}', '\u{552}', '\u{0}'], ['H', '\u{331}', '\u{0}'], ['T', '\u{308}', '\u{0}'],

src/tools/unicode-table-generator/src/case_mapping.rs

@@ -6,24 +6,26 @@ use crate::{UnicodeData, fmt_list};
 
 const INDEX_MASK: u32 = 1 << 22;
 
-pub(crate) fn generate_case_mapping(data: &UnicodeData) -> String {
+pub(crate) fn generate_case_mapping(data: &UnicodeData) -> (String, [usize; 2]) {
     let mut file = String::new();
 
     write!(file, "const INDEX_MASK: u32 = 0x{INDEX_MASK:x};").unwrap();
     file.push_str("\n\n");
     file.push_str(HEADER.trim_start());
     file.push('\n');
-    file.push_str(&generate_tables("LOWER", &data.to_lower));
+    let (lower_tables, lower_size) = generate_tables("LOWER", &data.to_lower);
+    file.push_str(&lower_tables);
     file.push_str("\n\n");
-    file.push_str(&generate_tables("UPPER", &data.to_upper));
-    file
+    let (upper_tables, upper_size) = generate_tables("UPPER", &data.to_upper);
+    file.push_str(&upper_tables);
+    (file, [lower_size, upper_size])
 }
 
-fn generate_tables(case: &str, data: &BTreeMap<u32, (u32, u32, u32)>) -> String {
+fn generate_tables(case: &str, data: &BTreeMap<u32, [u32; 3]>) -> (String, usize) {
     let mut mappings = Vec::with_capacity(data.len());
     let mut multis = Vec::new();
 
-    for (&key, &(a, b, c)) in data.iter() {
+    for (&key, &[a, b, c]) in data.iter() {
         let key = char::from_u32(key).unwrap();
 
         if key.is_ascii() {
@@ -46,16 +48,31 @@ fn generate_tables(case: &str, data: &BTreeMap<u32, (u32, u32, u32)>) -> String
     }
 
     let mut tables = String::new();
-
-    write!(tables, "static {}CASE_TABLE: &[(char, u32)] = &[{}];", case, fmt_list(mappings))
-        .unwrap();
+    let mut size = 0;
+
+    size += size_of_val(mappings.as_slice());
+    write!(
+        tables,
+        "static {}CASE_TABLE: &[(char, u32); {}] = &[{}];",
+        case,
+        mappings.len(),
+        fmt_list(mappings),
+    )
+    .unwrap();
 
     tables.push_str("\n\n");
 
-    write!(tables, "static {}CASE_TABLE_MULTI: &[[char; 3]] = &[{}];", case, fmt_list(multis))
-        .unwrap();
-
-    tables
+    size += size_of_val(multis.as_slice());
+    write!(
+        tables,
+        "static {}CASE_TABLE_MULTI: &[[char; 3]; {}] = &[{}];",
+        case,
+        multis.len(),
+        fmt_list(multis),
+    )
+    .unwrap();
+
+    (tables, size)
 }
 
 struct CharEscape(char);