Initial version of dynamically removing initial punct from sentences - lets us remove that from the dataset processing and lets the models train on all sentences with that executed dynamically.

Author: AngledLuffa

stanza/models/common/data.py

@@ -83,3 +83,45 @@ def get_augment_ratio(train_data, should_augment_predicate, can_augment_predicat
     if ratio > max_ratio:
         return max_ratio
     return ratio
+
+
+# Spanish/Catalan-style inverted question and exclamation marks. Some UD
+# treebanks have every training sentence begin with one of these, which
+# teaches a model to always expect the mark and misparse/mistag/mistokenize
+# a sentence that's missing it. augment_initial_punct in
+# prepare_tokenizer_treebank.py handles this at dataset-preparation time
+# (currently for ¿ only); the POS tagger and dependency parser instead
+# apply the equivalent augmentation dynamically, per sentence, inside their
+# own Dataset.__getitem__ (see starts_with_initial_mark below). The
+# tokenizer needs its own character-level version of this same check
+# (see drop_initial_punct in stanza.models.tokenization.data), since it
+# operates on individual characters before word boundaries exist, but
+# imports this same tuple so the mark set itself is defined in one place.
+INITIAL_INVERTED_PUNCT_MARKS = ('¿', '¡')
+
+def starts_with_initial_mark(words, marks=INITIAL_INVERTED_PUNCT_MARKS):
+    """
+    True if the given list of word/token strings starts with one of the
+    given marks, and no mark from the set (of any kind, not just the
+    leading one) appears anywhere else in the list.
+
+    The restriction mirrors augment_initial_punct in
+    prepare_tokenizer_treebank.py, and exists to avoid ambiguity with
+    nested or quoted questions/exclamations -- e.g. a sentence like
+    '¿Dijo "¡hola!"?' has two candidate marks (one ¿ and one ¡) and isn't
+    a case this augmentation should touch, even though neither mark is
+    individually repeated.
+
+    Shared by the POS tagger and dependency parser, whose sentences are
+    both, at this point, plain lists of word strings -- the two models
+    diverge only in how they physically drop the first word afterward
+    (the parser also has to renumber head positions, which the tagger
+    does not need to do).
+    """
+    if len(words) <= 1:
+        return False
+    first = words[0]
+    if first not in marks:
+        return False
+    total_marks = sum(1 for w in words if w in marks)
+    return total_marks == 1

stanza/models/depparse/data.py

@@ -7,7 +7,7 @@
 from torch.utils.data.sampler import Sampler
 
 from stanza.models.common.bert_embedding import filter_data, needs_length_filter
-from stanza.models.common.data import map_to_ids, get_long_tensor, get_float_tensor, sort_all, get_augment_ratio
+from stanza.models.common.data import map_to_ids, get_long_tensor, get_float_tensor, sort_all, get_augment_ratio, INITIAL_INVERTED_PUNCT_MARKS, starts_with_initial_mark
 from stanza.models.common.utils import DEFAULT_WORD_CUTOFF, simplify_punct
 from stanza.models.common.vocab import PAD_ID, VOCAB_PREFIX, ROOT_ID, CompositeVocab, CharVocab
 from stanza.models.pos.vocab import WordVocab, XPOSVocab, FeatureVocab, MultiVocab
@@ -104,6 +104,75 @@ def record_can_augment_nopunct(record, punct_id):
     return True
 
 
+def record_starts_with_mark(record, marks=INITIAL_INVERTED_PUNCT_MARKS):
+    """
+    True if this preprocessed sentence's first real word is exactly one
+    of the given marks (by default the Spanish/Catalan inverted question
+    and exclamation marks), and no mark from the set appears anywhere
+    else in the sentence.
+
+    Thin wrapper around stanza.models.common.data.starts_with_initial_mark
+    -- record[9] is the plain-string text field (never vocab-mapped),
+    which is exactly the "list of word strings" that function expects.
+    The actual mark-matching logic (including the no-other-mark
+    restriction, mirroring augment_initial_punct in
+    prepare_tokenizer_treebank.py) lives there in one place, shared with
+    the POS tagger's equivalent eligibility check; only this record[9]
+    extraction is specific to the parser's own record shape.
+    """
+    return starts_with_initial_mark(record[9], marks)
+
+
+def record_can_drop_initial_mark(record, marks=INITIAL_INVERTED_PUNCT_MARKS):
+    """
+    True if the sentence starts with one of the given marks (see
+    record_starts_with_mark) and no other word's head depends directly
+    on that first token -- removing it would otherwise leave a word's
+    head pointing at a position that no longer exists.
+
+    record[7] is the head field (no ROOT, one 1-indexed position per
+    real word); the first real word's own 1-indexed position is always 1.
+    """
+    if not record_starts_with_mark(record, marks):
+        return False
+    head = record[7]
+    if any(h == 1 for h in head[1:]):
+        return False
+    return True
+
+
+def drop_initial_mark_from_record(record):
+    """
+    Removes the first real word from a preprocessed sentence, renumbering
+    every remaining word's head position down by one to account for it.
+
+    Unlike dropping the last word (record_can_augment_nopunct's case),
+    dropping the FIRST word shifts every later word's 1-indexed position
+    back by one, so every head value greater than 0 must be decremented;
+    a head of 0 (root) is left untouched. record_can_drop_initial_mark
+    already guarantees no remaining word's head is exactly 1 (i.e.
+    nothing depends on the word being removed), so every nonzero head
+    among the remaining words is guaranteed to be >= 2 before the shift.
+    """
+    word, char, upos, xpos, feats, pretrain, lemma, head, deprel, text = record
+    # ROOT-prepended fields: keep ROOT (index 0), drop the removed word
+    # (index 1), keep everything after it unchanged
+    new_word = [word[0]] + word[2:]
+    new_char = [char[0]] + char[2:]
+    new_upos = [upos[0]] + upos[2:]
+    new_xpos = [xpos[0]] + xpos[2:]
+    new_feats = [feats[0]] + feats[2:]
+    new_pretrain = [pretrain[0]] + pretrain[2:]
+    new_lemma = [lemma[0]] + lemma[2:]
+    # non-ROOT-prepended fields: drop the removed word's own entry, and
+    # shift every remaining head position down by one
+    new_head = [h - 1 if h > 0 else h for h in head[1:]]
+    new_deprel = deprel[1:]
+    new_text = text[1:]
+    return [new_word, new_char, new_upos, new_xpos, new_feats, new_pretrain,
+            new_lemma, new_head, new_deprel, new_text]
+
+
 class Dataset:
     """
     Sentence-level dataset for the dependency parser: owns vocab
@@ -187,6 +256,23 @@ def __init__(self, doc, args, pretrain, vocab=None, evaluation=False, bert_token
         else:
             self.augment_nopunct_ratio = augment_nopunct_arg
 
+        # dynamic leading-inverted-punct drop: some UD treebanks (Spanish,
+        # Catalan) have every training sentence begin with an inverted
+        # question or exclamation mark (¿/¡), which the model never learns
+        # to do without. Mirrors augment_initial_punct in
+        # prepare_tokenizer_treebank.py, applied per sentence in
+        # __getitem__ instead of by duplicating sentences at dataset-
+        # preparation time. Unlike augment_nopunct, this uses a flat
+        # default ratio (not an auto-detected one), matching the ratio
+        # parameter augment_initial_punct itself takes.
+        drop_initial_punct_arg = args.get('drop_initial_punct_prob', 0.20)
+        if self.eval or not drop_initial_punct_arg or drop_initial_punct_arg <= 0:
+            self.drop_initial_punct_eligible = False
+            self.drop_initial_punct_ratio = 0.0
+        else:
+            self.drop_initial_punct_eligible = any(record_starts_with_mark(record) for record in self.data)
+            self.drop_initial_punct_ratio = drop_initial_punct_arg if self.drop_initial_punct_eligible else 0.0
+
     def init_vocab(self, data):
         assert self.eval == False # for eval vocab must exist
         cutoff = self.args['word_cutoff'] if self.args.get('word_cutoff') is not None else DEFAULT_WORD_CUTOFF
@@ -250,6 +336,11 @@ def __getitem__(self, key):
                 # lemma) and non-ROOT-prepended fields (head/deprel/text)
                 # all simply lose their last entry
                 record = [field[:-1] for field in record]
+        if self.drop_initial_punct_ratio > 0 and record_can_drop_initial_mark(record):
+            if random.random() < self.drop_initial_punct_ratio:
+                # drop the leading ¿/¡ and renumber every remaining word's
+                # head position down by one -- see drop_initial_mark_from_record
+                record = drop_initial_mark_from_record(record)
         return record
 
     def __iter__(self):

stanza/models/parser.py

@@ -240,6 +240,7 @@ def build_argparse():
     #
     # One simple way to fix this is to train on some fraction of training data with punct.
     parser.add_argument('--augment_nopunct', type=float, default=None, help='Fraction of punct-ending sentences to dynamically present without the final punct, applied fresh each epoch rather than by duplicating sentences in the training set.  Default of None will aim for roughly 10%%')
+    parser.add_argument('--drop_initial_punct_prob', type=float, default=0.20, help='Probability to drop a leading inverted question or exclamation mark (¿/¡) from a sentence, for languages such as Spanish and Catalan where it should be optional')
 
     parser.add_argument('--wandb', action='store_true', help='Start a wandb session and write the results of training.  Only applies to training.  Use --wandb_name instead to specify a name')
     parser.add_argument('--wandb_name', default=None, help='Name of a wandb session to start when training.  Will default to the dataset short name')

stanza/models/pos/data.py

@@ -9,7 +9,7 @@
 from torch.nn.utils.rnn import pad_sequence
 
 from stanza.models.common.bert_embedding import filter_data, needs_length_filter
-from stanza.models.common.data import map_to_ids, get_long_tensor, get_float_tensor, sort_all
+from stanza.models.common.data import map_to_ids, get_long_tensor, get_float_tensor, sort_all, starts_with_initial_mark
 from stanza.models.common.utils import DEFAULT_WORD_CUTOFF, simplify_punct
 from stanza.models.common.vocab import PAD_ID, VOCAB_PREFIX, CharVocab
 from stanza.models.pos.vocab import WordVocab, XPOSVocab, FeatureVocab, MultiVocab
@@ -54,6 +54,27 @@ def __init__(self, doc, args, pretrain, vocab=None, evaluation=False, sort_durin
             data = random.sample(data, keep)
             logger.debug("Subsample training set with rate {:g}".format(args['sample_train']))
 
+        # dynamic leading-¿/¡ drop: some UD treebanks (Spanish, Catalan) have
+        # every training sentence with a leading inverted question or
+        # exclamation mark, which the model never learns to do without.
+        #
+        # Eligibility is checked against the raw sentence data (before
+        # self.vocab['word'] is even queried), not by asking whether ¿/¡
+        # is IN self.vocab['word']. self.vocab['word'] is built with a
+        # frequency cutoff (DEFAULT_WORD_CUTOFF, or word_cutoff if set) --
+        # a word appearing fewer times than the cutoff is left out of the
+        # vocab and maps to UNK instead. In a small treebank, ¿/¡ could
+        # easily appear only a handful of times and fall under that
+        # cutoff, which would make a vocab-containment check wrongly say
+        # "ineligible" even though the mark is genuinely present in the
+        # data. Scanning the raw sentences directly avoids that failure
+        # mode, so the augmentation still triggers correctly even on
+        # small treebanks. starts_with_initial_mark is shared with the
+        # dependency parser's equivalent eligibility check.
+        self.drop_initial_punct_eligible = not self.eval and any(
+            starts_with_initial_mark([w[0] for w in sent]) for sent in data)
+        self.drop_initial_punct_ratio = args.get('drop_initial_punct_prob', 0.20) if self.drop_initial_punct_eligible else 0.0
+
         data = self.preprocess(data, self.vocab, self.pretrain_vocab, args)
 
         self.data = data
@@ -208,6 +229,25 @@ def __getitem__(self, key):
                 char = char[:mask] + char[mask+1:]
                 raw_text = raw_text[:mask] + raw_text[mask+1:]
 
+        # dynamic leading-¿/¡ drop (see drop_initial_punct_eligible in
+        # __init__). Unlike the trailing-punct mask above, this can't be
+        # done by masking a position in place: removing the FIRST word
+        # has to shift every later position back by one, so every field
+        # is sliced consistently rather than one element being replaced
+        # with a placeholder while the rest stay put.
+        if (self.drop_initial_punct_ratio > 0 and starts_with_initial_mark(raw_text)
+                and random.uniform(0, 1) < self.drop_initial_punct_ratio):
+            words = words[1:]
+            if upos is not None:
+                upos = upos[1:]
+            if xpos is not None:
+                xpos = xpos[1:]
+            if ufeats is not None:
+                ufeats = ufeats[1:]
+            pretrained = pretrained[1:]
+            char = char[1:]
+            raw_text = raw_text[1:]
+
         # get each character from the input sentnece
         # chars = [w for sent in char for w in sent]
 

stanza/models/tagger.py

@@ -117,6 +117,7 @@ def build_argparse():
     utils.add_device_args(parser)
 
     parser.add_argument('--augment_nopunct', type=float, default=None, help='Augment the training data by copying this fraction of punct-ending sentences as non-punct.  Default of None will aim for roughly 50%%')
+    parser.add_argument('--drop_initial_punct_prob', type=float, default=0.20, help='Probability to drop a leading inverted question or exclamation mark (¿/¡) from a sentence, for languages such as Spanish and Catalan where it should be optional')
 
     parser.add_argument('--wandb', action='store_true', help='Start a wandb session and write the results of training.  Only applies to training.  Use --wandb_name instead to specify a name')
     parser.add_argument('--wandb_name', default=None, help='Name of a wandb session to start when training.  Will default to the dataset short name')