improve numeric semiotic classes

Signed-off-by: folivoramanh <palasek182@gmail.com>

Author: folivoramanh

nemo_text_processing/inverse_text_normalization/vi/graph_utils.py

@@ -35,9 +35,10 @@
 NEMO_HEX = pynini.union(*string.hexdigits).optimize()
 NEMO_NON_BREAKING_SPACE = "\u00a0"
 NEMO_SPACE = " "
+NEMO_QUOTE = r'"'
 NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00a0").optimize()
 NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize()
-NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize()
+NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, NEMO_QUOTE).optimize()
 
 NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize()
 NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize()

nemo_text_processing/inverse_text_normalization/vi/taggers/cardinal.py

@@ -36,117 +36,124 @@ class CardinalFst(GraphFst):
 
     def __init__(self):
         super().__init__(name="cardinal", kind="classify")
-        graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
-        graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
+        self.graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
+        self.graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
         graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv"))
         graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv"))
 
-        graph_one = pynini.cross("mốt", "1")
-        graph_four = pynini.cross("tư", "4")
-        graph_five = pynini.cross("lăm", "5")
-        graph_half = pynini.cross("rưỡi", "5")
+        self.graph_one = pynini.cross("mốt", "1")
+        self.graph_four = pynini.cross("tư", "4")
+        self.graph_five = pynini.cross("lăm", "5")
+        self.graph_half = pynini.cross("rưỡi", "5")
+        
+        self.magnitude_words = pynini.union("triệu", "tỉ", "tỷ", "vạn")
+        self.thousand_words = pynini.union("ngàn", "nghìn")
+        self.negative_words = pynini.union("âm", "trừ")
+        
         graph_hundred = pynini.cross("trăm", "")
         graph_ten = pynini.cross("mươi", "")
         zero = pynini.cross(pynini.union("linh", "lẻ"), "0")
+        
+        graph_zero = self.graph_zero
+        graph_digit = self.graph_digit
+        graph_one = self.graph_one
+        graph_four = self.graph_four
+        graph_five = self.graph_five
+        graph_half = self.graph_half
 
         optional_ten = pynini.closure(delete_space + graph_ten, 0, 1)
         last_digit_exception = pynini.project(pynini.cross("năm", "5"), "input")
-        last_digit = pynini.union(
+        self.last_digit = pynini.union(
             (pynini.project(graph_digit, "input") - last_digit_exception.arcsort()) @ graph_digit,
             graph_one,
             graph_four,
             graph_five,
         )
-
-        graph_hundred_ties_component = (graph_digit | graph_zero) + delete_space + graph_hundred
-        graph_hundred_ties_component += delete_space
-        graph_hundred_ties_component += pynini.union(
+        last_digit = self.last_digit
+        # Build hundreds component (e.g., "một trăm", "hai trăm")
+        graph_hundreds_component = (graph_digit | graph_zero) + delete_space + graph_hundred
+        graph_hundreds_component += delete_space
+        graph_hundreds_component += pynini.union(
             graph_teen,
-            (graph_half | graph_four | graph_one) + pynutil.insert("0"),
-            graph_ties + optional_ten + ((delete_space + last_digit) | pynutil.insert("0")),
-            zero + delete_space + (graph_digit | graph_four),
-            pynutil.insert("00"),
-        )
-        graph_hundred_ties_component |= (
+            (graph_half | graph_four | graph_one) + pynutil.insert("0", weight=0.1),
+            graph_ties + optional_ten + ((delete_space + last_digit) | pynutil.insert("0", weight=0.1)),
+            zero + delete_space + (graph_digit | graph_four | graph_five),
+            pynutil.insert("00", weight=0.1),
+        ).optimize()
+        graph_hundreds_component |= (
             pynutil.insert("0")
             + delete_space
             + pynini.union(
                 graph_teen,
                 graph_ties + optional_ten + delete_space + last_digit,
-                graph_ties + delete_space + graph_ten + pynutil.insert("0"),
-                zero + delete_space + (graph_digit | graph_four),
-            )
+                graph_ties + delete_space + graph_ten + pynutil.insert("0", weight=0.1),
+                zero + delete_space + (graph_digit | graph_four | graph_five),
+            ).optimize()
         )
-        graph_hundred_component = graph_hundred_ties_component | (pynutil.insert("00") + delete_space + graph_digit)
+        graph_hundred_component = graph_hundreds_component | (pynutil.insert("00", weight=0.1) + delete_space + graph_digit)
 
         graph_hundred_component_at_least_one_none_zero_digit = graph_hundred_component @ (
             pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT)
         )
         self.graph_hundred_component_at_least_one_none_zero_digit = (
-            graph_hundred_component_at_least_one_none_zero_digit
+            graph_hundred_component_at_least_one_none_zero_digit.optimize()
         )
-        graph_hundred_ties_zero = graph_hundred_ties_component | pynutil.insert("000")
+        graph_hundreds_zero = graph_hundreds_component | pynutil.insert("000", weight=0.1)
 
         graph_thousands = pynini.union(
             graph_hundred_component_at_least_one_none_zero_digit
             + delete_space
-            + pynutil.delete(pynini.union("nghìn", "ngàn")),
+            + pynutil.delete(self.thousand_words),
             pynutil.insert("000", weight=0.1),
-        )
-
-        graph_ten_thousand = pynini.union(
-            graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete("vạn"),
-            pynutil.insert("0000", weight=0.1),
-        )
+        ).optimize()
 
-        graph_ten_thousand_suffix = pynini.union(
-            graph_digit + delete_space + pynutil.delete(pynini.union("nghìn", "ngàn")),
-            pynutil.insert("0", weight=0.1),
-        )
 
         graph_million = pynini.union(
             graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete("triệu"),
             pynutil.insert("000", weight=0.1),
-        )
+        ).optimize()
         graph_billion = pynini.union(
             graph_hundred_component_at_least_one_none_zero_digit
             + delete_space
             + pynutil.delete(pynini.union("tỉ", "tỷ")),
             pynutil.insert("000", weight=0.1),
-        )
+        ).optimize()
 
+        # Main graph combining all magnitude levels
         graph = pynini.union(
+            # Full format: billion + million + thousand + hundred  
             graph_billion
             + delete_space
             + graph_million
             + delete_space
             + graph_thousands
             + delete_space
-            + graph_hundred_ties_zero,
-            graph_ten_thousand + delete_space + graph_ten_thousand_suffix + delete_space + graph_hundred_ties_zero,
+            + graph_hundreds_zero,
+            # Special thousand format with last digit or "rưỡi" (half)
             graph_hundred_component_at_least_one_none_zero_digit
             + delete_space
-            + pynutil.delete(pynini.union("nghìn", "ngàn"))
+            + pynutil.delete(self.thousand_words)
             + delete_space
-            + (((last_digit | graph_half) + pynutil.insert("00")) | graph_hundred_ties_zero),
+            + (((last_digit | graph_half) + pynutil.insert("00", weight=0.1)) | graph_hundreds_zero),
+            # Single digits (for non-exception cases)
             graph_digit,
             graph_zero,
         )
 
         graph = graph @ pynini.union(
             pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT),
             "0",
-        )
+        ).optimize()
 
         # don't convert cardinals from zero to nine inclusive
-        graph_exception = pynini.project(pynini.union(graph_digit, graph_zero), "input")
+        single_digits = pynini.project(pynini.union(graph_digit, graph_zero), "input").optimize()
 
         self.graph_no_exception = graph
 
-        self.graph = (pynini.project(graph, "input") - graph_exception.arcsort()) @ graph
+        self.graph = pynini.difference(pynini.project(graph, "input"), single_digits) @ graph
 
         optional_minus_graph = pynini.closure(
-            pynutil.insert("negative: ") + pynini.cross(pynini.union("âm", "trừ"), '"-"') + NEMO_SPACE,
+            pynutil.insert("negative: ") + pynini.cross(self.negative_words, '"-"') + NEMO_SPACE,
             0,
             1,
         )

nemo_text_processing/inverse_text_normalization/vi/taggers/decimal.py

@@ -21,64 +21,9 @@
     GraphFst,
     delete_extra_space,
     delete_space,
+    NEMO_QUOTE,
+    insert_space
 )
-from nemo_text_processing.inverse_text_normalization.vi.utils import get_abs_path
-
-graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
-
-
-def get_quantity(decimal: "pynini.FstLike", cardinal_up_to_hundred: "pynini.FstLike") -> "pynini.FstLike":
-    """
-    Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral,
-    e.g. một triệu -> integer_part: "1" quantity: "triệu"
-    e.g. một tỷ rưỡi -> integer_part: "1" fractional_part: "5" quantity: "tỷ"
-
-    Args:
-        decimal: decimal FST
-        cardinal_up_to_hundred: cardinal FST
-    """
-    numbers = cardinal_up_to_hundred @ (
-        pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT)
-    )
-    suffix = pynini.union("triệu", "tỉ", "tỷ", "vạn")
-    graph_four = pynini.cross("tư", "4")
-    graph_one = pynini.cross("mốt", "1")
-    graph_half = pynini.cross("rưỡi", "5")
-    last_digit_exception = pynini.project(pynini.cross("năm", "5"), "input")
-    last_digit = pynini.union(
-        (pynini.project(graph_digit, "input") - last_digit_exception.arcsort()) @ graph_digit,
-        graph_one,
-        graph_four,
-        graph_half,
-    )
-    optional_fraction_graph = pynini.closure(
-        delete_extra_space
-        + pynutil.insert('fractional_part: "')
-        + (last_digit | graph_half | graph_one | graph_four)
-        + pynutil.insert('"'),
-        0,
-        1,
-    )
-
-    res = (
-        pynutil.insert('integer_part: "')
-        + numbers
-        + pynutil.insert('"')
-        + delete_extra_space
-        + pynutil.insert('quantity: "')
-        + suffix
-        + pynutil.insert('"')
-        + optional_fraction_graph
-    )
-    res |= (
-        decimal
-        + delete_extra_space
-        + pynutil.insert('quantity: "')
-        + (suffix | "ngàn" | "nghìn")
-        + pynutil.insert('"')
-    )
-    return res
-
 
 class DecimalFst(GraphFst):
     """
@@ -95,40 +40,67 @@ def __init__(self, cardinal: GraphFst):
 
         cardinal_graph = cardinal.graph_no_exception
 
-        graph_decimal = graph_digit | pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
-        graph_one = pynini.cross("mốt", "1")
-        graph_four = pynini.cross("tư", "4")
-        graph_five = pynini.cross("lăm", "5")
-
+        base_decimal = cardinal.graph_digit | cardinal.graph_zero
         graph_decimal = pynini.union(
-            graph_decimal,
-            graph_four,
-            pynini.closure(graph_decimal + delete_space, 1) + (graph_decimal | graph_four | graph_five | graph_one),
-        )
+            base_decimal,
+            cardinal.graph_four,
+            pynini.closure(base_decimal + delete_space, 1) + (base_decimal | cardinal.graph_four | cardinal.graph_five | cardinal.graph_one),
+        ).optimize()
         self.graph = graph_decimal
 
         point = pynutil.delete("chấm") | pynutil.delete("phẩy")
-
         optional_graph_negative = pynini.closure(
-            pynutil.insert("negative: ") + pynini.cross(pynini.union("âm", "trừ"), '"true"') + delete_extra_space,
+            pynutil.insert("negative:") + insert_space + pynini.cross(cardinal.negative_words, '"true"') + delete_extra_space,
             0,
             1,
         )
 
-        graph_fractional = pynutil.insert('fractional_part: "') + graph_decimal + pynutil.insert('"')
-        graph_integer = pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"')
+        graph_fractional = pynutil.insert('fractional_part:') + insert_space + pynutil.insert(NEMO_QUOTE) + graph_decimal + pynutil.insert(NEMO_QUOTE)
+        graph_integer = pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert(NEMO_QUOTE)
         final_graph_wo_sign = (
             pynini.closure(graph_integer + delete_extra_space, 0, 1) + point + delete_extra_space + graph_fractional
         )
-        final_graph = optional_graph_negative + final_graph_wo_sign
+        # Build quantity handling - reuse magnitude words from cardinal context
+        # e.g. một triệu -> integer_part: "1" quantity: "triệu"  
+        # e.g. một tỷ rưỡi -> integer_part: "1" fractional_part: "5" quantity: "tỷ"
+        numbers = cardinal.graph_hundred_component_at_least_one_none_zero_digit @ (
+            pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT)
+        )
+        
+        magnitude_words = cardinal.magnitude_words
+        thousand_words = cardinal.thousand_words
+        
+        last_digit = cardinal.last_digit
+        optional_fraction_graph = pynini.closure(
+            delete_extra_space
+            + pynutil.insert('fractional_part:') + insert_space + pynutil.insert(NEMO_QUOTE)
+            + (last_digit | cardinal.graph_half | cardinal.graph_one | cardinal.graph_four)
+            + pynutil.insert(NEMO_QUOTE),
+            0,
+            1,
+        )
 
-        self.final_graph_wo_negative = final_graph_wo_sign | get_quantity(
-            final_graph_wo_sign,
-            cardinal.graph_hundred_component_at_least_one_none_zero_digit,
+        quantity_graph = (
+            pynutil.insert('integer_part:') + insert_space + pynutil.insert(NEMO_QUOTE)
+            + numbers
+            + pynutil.insert(NEMO_QUOTE)
+            + delete_extra_space
+            + pynutil.insert('quantity:') + insert_space + pynutil.insert(NEMO_QUOTE)
+            + magnitude_words
+            + pynutil.insert(NEMO_QUOTE)
+            + optional_fraction_graph
         )
-        final_graph |= optional_graph_negative + get_quantity(
-            final_graph_wo_sign,
-            cardinal.graph_hundred_component_at_least_one_none_zero_digit,
+        quantity_graph |= (
+            final_graph_wo_sign
+            + delete_extra_space
+            + pynutil.insert('quantity:') + insert_space + pynutil.insert(NEMO_QUOTE)
+            + (magnitude_words | thousand_words)
+            + pynutil.insert(NEMO_QUOTE)
         )
+
+        final_graph = optional_graph_negative + final_graph_wo_sign
+
+        self.final_graph_wo_negative = final_graph_wo_sign | quantity_graph
+        final_graph |= optional_graph_negative + quantity_graph
         final_graph = self.add_tokens(final_graph)
         self.fst = final_graph.optimize()

nemo_text_processing/inverse_text_normalization/vi/taggers/fraction.py

@@ -16,7 +16,10 @@
 import pynini
 from pynini.lib import pynutil
 
-from nemo_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst, delete_extra_space, delete_space
+from nemo_text_processing.inverse_text_normalization.vi.graph_utils import (
+    GraphFst,
+    delete_extra_space, delete_space, NEMO_QUOTE, insert_space
+)
 
 
 class FractionFst(GraphFst):
@@ -32,14 +35,13 @@ class FractionFst(GraphFst):
 
     def __init__(self, cardinal: GraphFst):
         super().__init__(name="fraction", kind="classify")
-        # integer_part # numerator # denominator
 
         graph_cardinal = cardinal.graph_no_exception
-        graph_four = pynini.cross("tư", "4")
+        graph_four = cardinal.graph_four
 
-        numerator = pynutil.insert('numerator: "') + graph_cardinal + pynutil.insert('"')
+        numerator = pynutil.insert('numerator:') + insert_space + pynutil.insert(NEMO_QUOTE) + graph_cardinal + pynutil.insert(NEMO_QUOTE)
         fraction_component = pynutil.delete(pynini.union("phần", "trên", "chia"))
-        denominator = pynutil.insert('denominator: "') + (graph_cardinal | graph_four) + pynutil.insert('"')
+        denominator = pynutil.insert('denominator:') + insert_space + pynutil.insert(NEMO_QUOTE) + (graph_cardinal | graph_four) + pynutil.insert(NEMO_QUOTE)
 
         graph_fraction_component = numerator + delete_space + fraction_component + delete_extra_space + denominator
         self.graph_fraction_component = graph_fraction_component
@@ -49,7 +51,7 @@ def __init__(self, cardinal: GraphFst):
         self.final_graph_wo_negative = graph
 
         optional_graph_negative = pynini.closure(
-            pynutil.insert("negative: ") + pynini.cross(pynini.union("âm", "trừ"), '"true"') + delete_extra_space,
+            pynutil.insert("negative:") + insert_space + pynini.cross(pynini.union("âm", "trừ"), '"true"') + delete_extra_space,
             0,
             1,
         )

nemo_text_processing/inverse_text_normalization/vi/taggers/ordinal.py

@@ -16,7 +16,9 @@
 import pynini
 from pynini.lib import pynutil
 
-from nemo_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst, delete_space
+from nemo_text_processing.inverse_text_normalization.vi.graph_utils import (
+    GraphFst,delete_space, NEMO_QUOTE, insert_space
+)   
 from nemo_text_processing.inverse_text_normalization.vi.utils import get_abs_path
 
 
@@ -34,6 +36,6 @@ def __init__(self):
         graph = graph_digit
 
         self.graph = graph
-        final_graph = pynutil.insert('integer: "') + graph_ordinal + delete_space + self.graph + pynutil.insert('"')
+        final_graph = pynutil.insert('integer:') + insert_space + pynutil.insert(NEMO_QUOTE) + graph_ordinal + delete_space + self.graph + pynutil.insert(NEMO_QUOTE)
         final_graph = self.add_tokens(final_graph)
         self.fst = final_graph.optimize()