Redesign combining table

eschnett · eschnett · commit b18c5b522770 · 2024-12-18T15:55:35.000-05:00
diff --git a/data/Manifest.toml b/data/Manifest.toml
@@ -1,13 +1,13 @@
 # This file is machine-generated - editing it directly is not advised
 
-julia_version = "1.10.5"
+julia_version = "1.11.2"
 manifest_format = "2.0"
 project_hash = "bc0740aa2247b17bd49ba693fb87f41bbbddead6"
 
 [[deps.OffsetArrays]]
-git-tree-sha1 = "1a27764e945a152f7ca7efa04de513d473e9542e"
+git-tree-sha1 = "5e1897147d1ff8d98883cda2be2187dcf57d8f0c"
 uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881"
-version = "1.14.1"
+version = "1.15.0"
 
     [deps.OffsetArrays.extensions]
     OffsetArraysAdaptExt = "Adapt"
diff --git a/data/data_generator.jl b/data/data_generator.jl
@@ -256,82 +256,99 @@ end
 # decompressed on the C side at runtime.
 
 # Inverse decomposition mapping tables for combining two characters into a single one.
-comb1st_indices = Dict{UInt32,Int}()
-comb1st_indices_sorted_keys = Origin(0)(UInt32[])
-comb2nd_indices = Dict{UInt32,Int}()
-comb2nd_indices_sorted_keys = Origin(0)(UInt32[])
-comb2nd_indices_nonbasic = Set{UInt32}()
-comb_array = Origin(0)(Vector{Dict{Int,UInt32}}())
+comb_mapping = Dict{UInt32, Dict{UInt32, UInt32}}()
+comb_issecond = Set{UInt32}()
 for char in char_props
     if isnothing(char.decomp_type) && !isnothing(char.decomp_mapping) &&
             length(char.decomp_mapping) == 2 && !isnothing(char_hash[char.decomp_mapping[1]]) &&
             char_hash[char.decomp_mapping[1]].combining_class == 0 &&
             char.code ∉ exclusions
         dm0 = char.decomp_mapping[1]
         dm1 = char.decomp_mapping[2]
-        if !haskey(comb1st_indices, dm0)
-            comb1st_indices[dm0] = length(comb1st_indices)
-            push!(comb1st_indices_sorted_keys, dm0)
-            push!(comb_array, Dict{Int,UInt32}())
-            @assert length(comb1st_indices) == length(comb_array)
-        end
-        if !haskey(comb2nd_indices, dm1)
-            push!(comb2nd_indices_sorted_keys, dm1)
-            comb2nd_indices[dm1] = length(comb2nd_indices)
-        end
-        @assert !haskey(comb_array[comb1st_indices[dm0]], comb2nd_indices[dm1])
-        comb_array[comb1st_indices[dm0]][comb2nd_indices[dm1]] = char.code
-        if char.code > 0xFFFF
-            push!(comb2nd_indices_nonbasic, dm1)
+        if !haskey(comb_mapping, dm0)
+            comb_mapping[dm0] = Dict{UInt32, UInt32}()
         end
+        comb_mapping[dm0][dm1] = char.code
+        push!(comb_issecond, dm1)
     end
 end
 
-comb_indices = Dict{UInt32,Int}()
-comb1st_indices_lastoffsets = Origin(0)(zeros(Int, length(comb1st_indices)))
-comb1st_indices_firstoffsets = Origin(0)(zeros(Int, length(comb1st_indices)))
+comb_index = Dict{UInt32, UInt32}()
+comb_length = Dict{UInt32, UInt32}()
 let
-    cumoffset = 0
-    for dm0 in comb1st_indices_sorted_keys
-        index = comb1st_indices[dm0]
-        first = nothing
-        last = nothing
-        offset = 0
-        for b in eachindex(comb2nd_indices_sorted_keys)
-            dm1 = comb2nd_indices_sorted_keys[b]
-            if haskey(comb_array[index], b)
-                if isnothing(first)
-                    first = offset
-                end
-                last = offset
-                if dm1 in comb2nd_indices_nonbasic
-                    last += 1
-                end
-            end
-            offset += 1
-            if dm1 in comb2nd_indices_nonbasic
-                offset += 1
-            end
-        end
-        comb1st_indices_firstoffsets[index] = first
-        comb1st_indices_lastoffsets[index] = last
-        @assert !haskey(comb_indices, dm0)
-        comb_indices[dm0] = cumoffset
-        cumoffset += last - first + 1 + 2
-    end
-
-    offset = 0
-    for dm1 in comb2nd_indices_sorted_keys
-        @assert !haskey(comb_indices, dm1)
-        comb_indices[dm1] = 0x8000 | (comb2nd_indices[dm1] + offset)
-        @assert comb2nd_indices[dm1] + offset <= 0x4000
-        if dm1 in comb2nd_indices_nonbasic
-            comb_indices[dm1] |= 0x4000
-            offset += 1
-        end
+    ind = 0
+    for dm0 in sort!(collect(keys(comb_mapping)))
+        comb_index[dm0] = ind
+        len = length(comb_mapping[dm0])
+        comb_length[dm0] = len
+        ind += len
     end
 end
 
+# comb1st_indices = Dict{UInt32,Int}()
+# comb1st_indices_sorted_keys = Origin(0)(UInt32[])
+# comb2nd_indices = Dict{UInt32,Int}()
+# comb2nd_indices_sorted_keys = Origin(0)(UInt32[])
+# comb2nd_indices_length(code::UInt32) = code < 0x8000 ? 1 : 2
+# comb_array = Origin(0)(Vector{Dict{Int,UInt32}}())
+# for (i,char) in enumerate(char_props)
+#     if isnothing(char.decomp_type) && !isnothing(char.decomp_mapping) &&
+#             length(char.decomp_mapping) == 2 && !isnothing(char_hash[char.decomp_mapping[1]]) &&
+#             char_hash[char.decomp_mapping[1]].combining_class == 0 &&
+#             char.code ∉ exclusions
+#         dm0 = char.decomp_mapping[1]
+#         dm1 = char.decomp_mapping[2]
+#         if !haskey(comb1st_indices, dm0)
+#             comb1st_indices[dm0] = length(comb1st_indices)
+#             push!(comb1st_indices_sorted_keys, dm0)
+#             push!(comb_array, Dict{Int,UInt32}())
+#             @assert length(comb1st_indices) == length(comb_array)
+#         end
+#         if !haskey(comb2nd_indices, dm1)
+#             push!(comb2nd_indices_sorted_keys, dm1)
+#             comb2nd_indices[dm1] = length(comb2nd_indices)
+#         end
+#         @assert !haskey(comb_array[comb1st_indices[dm0]], comb2nd_indices[dm1])
+#         comb_array[comb1st_indices[dm0]][comb2nd_indices[dm1]] = char.code
+#     end
+# end
+# 
+# comb_indices = Dict{UInt32,Int}()
+# comb1st_indices_lastoffsets = Origin(0)(zeros(Int, length(comb1st_indices)))
+# comb1st_indices_firstoffsets = Origin(0)(zeros(Int, length(comb1st_indices)))
+# let
+#     cumoffset = 0
+#     for dm0 in comb1st_indices_sorted_keys
+#         index = comb1st_indices[dm0]
+#         first = nothing
+#         last = nothing
+#         offset = 0
+#         for b in eachindex(comb2nd_indices_sorted_keys)
+#             dm1 = comb2nd_indices_sorted_keys[b]
+#             if haskey(comb_array[index], b)
+#                 if isnothing(first)
+#                     first = offset
+#                 end
+#                 last = offset + comb2nd_indices_length(dm1) - 1
+#             end
+#             offset += comb2nd_indices_length(dm1)
+#         end
+#         comb1st_indices_firstoffsets[index] = first
+#         comb1st_indices_lastoffsets[index] = last
+#         @assert !haskey(comb_indices, dm0)
+#         comb_indices[dm0] = 0x4000 | cumoffset
+#         cumoffset += last - first + 1 + 2
+#     end
+# 
+#     offset = 0
+#     for dm1 in comb2nd_indices_sorted_keys
+#         @assert !haskey(comb_indices, dm1)
+#         comb_indices[dm1] = 0x8000 | (comb2nd_indices[dm1] + offset)
+#         @assert comb2nd_indices[dm1] + offset < 0x4000
+#         offset += comb2nd_indices_length(dm1) - 1
+#     end
+# end
+
 utf16_encode(utf32_seq) = transcode(UInt16, transcode(String, utf32_seq))
 
 # Utility for packing all UTF-16 encoded sequences into one big array
@@ -391,7 +408,9 @@ function char_table_properties!(sequences, char)
         uppercase_seqindex   = encode_sequence!(sequences, char.uppercase_mapping),
         lowercase_seqindex   = encode_sequence!(sequences, char.lowercase_mapping),
         titlecase_seqindex   = encode_sequence!(sequences, char.titlecase_mapping),
-        comb_index           = get(comb_indices, code, typemax(UInt16)),
+        comb_index           = get(comb_index, code, 0x3FF), # see utf8proc_property_struct::comb_index
+        comb_length          = get(comb_length, code, 0),
+        comb_issecond        = code in comb_issecond,
         bidi_mirrored        = char.bidi_mirrored,
         comp_exclusion       = code in exclusions || code in excl_version,
         ignorable            = code in ignorable,
@@ -473,8 +492,7 @@ function c_uint16(seqindex)
 end
 
 function print_c_data_tables(io, sequences, prop_page_indices, prop_pages, deduplicated_props,
-                             comb1st_indices_firstoffsets, comb1st_indices_lastoffsets,
-                             comb2nd_indices_sorted_keys, comb_array, comb2nd_indices_nonbasic)
+                             comb_index, comb_length, comb_issecond)
     print(io, "static const utf8proc_uint16_t utf8proc_sequences[] = ")
     write_c_index_array(io, sequences.storage, 8)
     print(io, "static const utf8proc_uint16_t utf8proc_stage1table[] = ")
@@ -484,7 +502,7 @@ function print_c_data_tables(io, sequences, prop_page_indices, prop_pages, dedup
 
     print(io, """
         static const utf8proc_property_t utf8proc_properties[] = {
-          {0, 0, 0, 0, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,  false,false,false,false, 1, 0, 0, UTF8PROC_BOUNDCLASS_OTHER, UTF8PROC_INDIC_CONJUNCT_BREAK_NONE},
+          {0, 0, 0, 0, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,  0x3FF,0,false,  false,false,false,false, 1, 0, 0, UTF8PROC_BOUNDCLASS_OTHER, UTF8PROC_INDIC_CONJUNCT_BREAK_NONE},
         """)
     for prop in deduplicated_props
         print(io, "  {",
@@ -498,6 +516,8 @@ function print_c_data_tables(io, sequences, prop_page_indices, prop_pages, dedup
               c_uint16(prop.lowercase_seqindex), ", ",
               c_uint16(prop.titlecase_seqindex), ", ",
               c_uint16(prop.comb_index), ", ",
+              c_uint16(prop.comb_length), ", ",
+              prop.comb_issecond, ", ",
               prop.bidi_mirrored, ", ",
               prop.comp_exclusion, ", ",
               prop.ignorable, ", ",
@@ -512,41 +532,18 @@ function print_c_data_tables(io, sequences, prop_page_indices, prop_pages, dedup
     end
     print(io, "};\n\n")
 
-    print(io, "static const utf8proc_uint16_t utf8proc_combinations[] = {\n  ")
-    i = 0
-    for a in eachindex(comb1st_indices_firstoffsets)
-        offset = 0
-        print(io, comb1st_indices_firstoffsets[a], ", ", comb1st_indices_lastoffsets[a], ", ")
-        for b in eachindex(comb2nd_indices_sorted_keys)
-            dm1 = comb2nd_indices_sorted_keys[b]
-            if offset > comb1st_indices_lastoffsets[a]
-                break
-            end
-            if offset >= comb1st_indices_firstoffsets[a]
-                i += 1
-                if i == 8
-                    i = 0
-                    print(io, "\n  ")
-                end
-                v = get(comb_array[a], b, 0)
-                if dm1 in comb2nd_indices_nonbasic
-                    print(io, (v & 0xFFFF0000) >> 16, ", ")
-                end
-                print(io, v & 0xFFFF, ", ")
-            end
-            offset += 1
-            if dm1 in comb2nd_indices_nonbasic
-                offset += 1
-            end
+    print(io, "static const utf8proc_uint32_t utf8proc_combinations[][2] = {\n")
+    for dm0 in sort!(collect(keys(comb_mapping)))
+        for dm1 in sort!(collect(keys(comb_mapping[dm0])))
+            code = comb_mapping[dm0][dm1]
+            print(io, "  { ", dm1, ", ", code, " },\n")
         end
-        print(io, "\n")
     end
     print(io, "};\n\n")
 end
 
 
 if !isinteractive()
     print_c_data_tables(stdout, sequences, prop_page_indices, prop_pages, deduplicated_props,
-                        comb1st_indices_firstoffsets, comb1st_indices_lastoffsets,
-                        comb2nd_indices_sorted_keys, comb_array, comb2nd_indices_nonbasic)
+                        comb_index, comb_length, comb_issecond)
 end
diff --git a/utf8proc.c b/utf8proc.c
@@ -651,7 +651,6 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *b
     utf8proc_propval_t max_combining_class = -1;
     utf8proc_ssize_t rpos;
     utf8proc_ssize_t wpos = 0;
-    utf8proc_int32_t composition;
     for (rpos = 0; rpos < length; rpos++) {
       current_char = buffer[rpos];
       current_property = unsafe_get_property(current_char);
@@ -685,22 +684,28 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *b
         if (!starter_property) {
           starter_property = unsafe_get_property(*starter);
         }
-        if (starter_property->comb_index < 0x8000 &&
-            current_property->comb_index != UINT16_MAX &&
-            current_property->comb_index >= 0x8000) {
-          int sidx = starter_property->comb_index;
-          int idx = current_property->comb_index & 0x3FFF;
-          if (idx >= utf8proc_combinations[sidx] && idx <= utf8proc_combinations[sidx + 1] ) {
-            idx += sidx + 2 - utf8proc_combinations[sidx];
-            if (current_property->comb_index & 0x4000) {
-              composition = (utf8proc_combinations[idx] << 16) | utf8proc_combinations[idx+1];
-            } else
-              composition = utf8proc_combinations[idx];
-
-            if (composition > 0 && (!(options & UTF8PROC_STABLE) ||
-                !(unsafe_get_property(composition)->comp_exclusion))) {
-              *starter = composition;
-              starter_property = NULL;
+        int idx = starter_property->comb_index;
+        if (idx < 0x3FF && current_property->comb_issecond) {
+          int len = starter_property->comb_length;
+          utf8proc_int32_t max_second = utf8proc_combinations[idx + len - 1][0];
+          if (current_char <= max_second) {
+            // TODO: binary search? arithmetic search?
+            for (int off = 0; off < len; ++off) {
+              utf8proc_int32_t second = utf8proc_combinations[idx + off][0];              
+              if (current_char < second) {
+                /* not found */
+                break;
+              }
+              if (current_char == second) {
+                /* found */
+                utf8proc_int32_t composition = utf8proc_combinations[idx + off][1];
+                *starter = composition;
+                starter_property = NULL;
+                break;
+              }
+            }
+            if (starter_property == NULL) {
+              /* found */
               continue;
             }
           }
diff --git a/utf8proc.h b/utf8proc.h
@@ -255,7 +255,9 @@ typedef struct utf8proc_property_struct {
   utf8proc_uint16_t uppercase_seqindex;
   utf8proc_uint16_t lowercase_seqindex;
   utf8proc_uint16_t titlecase_seqindex;
-  utf8proc_uint16_t comb_index;
+  utf8proc_uint16_t comb_index:10;
+  utf8proc_uint16_t comb_length:5;
+  utf8proc_uint16_t comb_issecond:1;
   unsigned bidi_mirrored:1;
   unsigned comp_exclusion:1;
   /**
diff --git a/utf8proc_data.c b/utf8proc_data.c
