Dynamically removing initial punct from sentences in each of tokenizer, pos, depparse - lets us remove that from the dataset processing and lets the models train on all sentences with that executed dynamically.

Turns out the tokenizer treats sentences after the first sentence in a doc as starting with a space.  The initial punct augmentation needs to respect that

Author: AngledLuffa

stanza/models/common/data.py

@@ -83,3 +83,45 @@ def get_augment_ratio(train_data, should_augment_predicate, can_augment_predicat
     if ratio > max_ratio:
         return max_ratio
     return ratio
+
+
+# Spanish/Catalan-style inverted question and exclamation marks. Some UD
+# treebanks have every training sentence begin with one of these, which
+# teaches a model to always expect the mark and misparse/mistag/mistokenize
+# a sentence that's missing it. augment_initial_punct in
+# prepare_tokenizer_treebank.py handles this at dataset-preparation time
+# (currently for ¿ only); the POS tagger and dependency parser instead
+# apply the equivalent augmentation dynamically, per sentence, inside their
+# own Dataset.__getitem__ (see starts_with_initial_mark below). The
+# tokenizer needs its own character-level version of this same check
+# (see drop_initial_punct in stanza.models.tokenization.data), since it
+# operates on individual characters before word boundaries exist, but
+# imports this same tuple so the mark set itself is defined in one place.
+INITIAL_INVERTED_PUNCT_MARKS = ('¿', '¡')
+
+def starts_with_initial_mark(words, marks=INITIAL_INVERTED_PUNCT_MARKS):
+    """
+    True if the given list of word/token strings starts with one of the
+    given marks, and no mark from the set (of any kind, not just the
+    leading one) appears anywhere else in the list.
+
+    The restriction mirrors augment_initial_punct in
+    prepare_tokenizer_treebank.py, and exists to avoid ambiguity with
+    nested or quoted questions/exclamations -- e.g. a sentence like
+    '¿Dijo "¡hola!"?' has two candidate marks (one ¿ and one ¡) and isn't
+    a case this augmentation should touch, even though neither mark is
+    individually repeated.
+
+    Shared by the POS tagger and dependency parser, whose sentences are
+    both, at this point, plain lists of word strings -- the two models
+    diverge only in how they physically drop the first word afterward
+    (the parser also has to renumber head positions, which the tagger
+    does not need to do).
+    """
+    if len(words) <= 1:
+        return False
+    first = words[0]
+    if first not in marks:
+        return False
+    total_marks = sum(1 for w in words if w in marks)
+    return total_marks == 1

stanza/models/depparse/data.py

@@ -7,7 +7,7 @@
 from torch.utils.data.sampler import Sampler
 
 from stanza.models.common.bert_embedding import filter_data, needs_length_filter
-from stanza.models.common.data import map_to_ids, get_long_tensor, get_float_tensor, sort_all, get_augment_ratio
+from stanza.models.common.data import map_to_ids, get_long_tensor, get_float_tensor, sort_all, get_augment_ratio, INITIAL_INVERTED_PUNCT_MARKS, starts_with_initial_mark
 from stanza.models.common.utils import DEFAULT_WORD_CUTOFF, simplify_punct
 from stanza.models.common.vocab import PAD_ID, VOCAB_PREFIX, ROOT_ID, CompositeVocab, CharVocab
 from stanza.models.pos.vocab import WordVocab, XPOSVocab, FeatureVocab, MultiVocab
@@ -104,6 +104,75 @@ def record_can_augment_nopunct(record, punct_id):
     return True
 
 
+def record_starts_with_mark(record, marks=INITIAL_INVERTED_PUNCT_MARKS):
+    """
+    True if this preprocessed sentence's first real word is exactly one
+    of the given marks (by default the Spanish/Catalan inverted question
+    and exclamation marks), and no mark from the set appears anywhere
+    else in the sentence.
+
+    Thin wrapper around stanza.models.common.data.starts_with_initial_mark
+    -- record[9] is the plain-string text field (never vocab-mapped),
+    which is exactly the "list of word strings" that function expects.
+    The actual mark-matching logic (including the no-other-mark
+    restriction, mirroring augment_initial_punct in
+    prepare_tokenizer_treebank.py) lives there in one place, shared with
+    the POS tagger's equivalent eligibility check; only this record[9]
+    extraction is specific to the parser's own record shape.
+    """
+    return starts_with_initial_mark(record[9], marks)
+
+
+def record_can_drop_initial_mark(record, marks=INITIAL_INVERTED_PUNCT_MARKS):
+    """
+    True if the sentence starts with one of the given marks (see
+    record_starts_with_mark) and no other word's head depends directly
+    on that first token -- removing it would otherwise leave a word's
+    head pointing at a position that no longer exists.
+
+    record[7] is the head field (no ROOT, one 1-indexed position per
+    real word); the first real word's own 1-indexed position is always 1.
+    """
+    if not record_starts_with_mark(record, marks):
+        return False
+    head = record[7]
+    if any(h == 1 for h in head[1:]):
+        return False
+    return True
+
+
+def drop_initial_mark_from_record(record):
+    """
+    Removes the first real word from a preprocessed sentence, renumbering
+    every remaining word's head position down by one to account for it.
+
+    Unlike dropping the last word (record_can_augment_nopunct's case),
+    dropping the FIRST word shifts every later word's 1-indexed position
+    back by one, so every head value greater than 0 must be decremented;
+    a head of 0 (root) is left untouched. record_can_drop_initial_mark
+    already guarantees no remaining word's head is exactly 1 (i.e.
+    nothing depends on the word being removed), so every nonzero head
+    among the remaining words is guaranteed to be >= 2 before the shift.
+    """
+    word, char, upos, xpos, feats, pretrain, lemma, head, deprel, text = record
+    # ROOT-prepended fields: keep ROOT (index 0), drop the removed word
+    # (index 1), keep everything after it unchanged
+    new_word = [word[0]] + word[2:]
+    new_char = [char[0]] + char[2:]
+    new_upos = [upos[0]] + upos[2:]
+    new_xpos = [xpos[0]] + xpos[2:]
+    new_feats = [feats[0]] + feats[2:]
+    new_pretrain = [pretrain[0]] + pretrain[2:]
+    new_lemma = [lemma[0]] + lemma[2:]
+    # non-ROOT-prepended fields: drop the removed word's own entry, and
+    # shift every remaining head position down by one
+    new_head = [h - 1 if h > 0 else h for h in head[1:]]
+    new_deprel = deprel[1:]
+    new_text = text[1:]
+    return [new_word, new_char, new_upos, new_xpos, new_feats, new_pretrain,
+            new_lemma, new_head, new_deprel, new_text]
+
+
 class Dataset:
     """
     Sentence-level dataset for the dependency parser: owns vocab
@@ -187,6 +256,23 @@ def __init__(self, doc, args, pretrain, vocab=None, evaluation=False, bert_token
         else:
             self.augment_nopunct_ratio = augment_nopunct_arg
 
+        # dynamic leading-inverted-punct drop: some UD treebanks (Spanish,
+        # Catalan) have every training sentence begin with an inverted
+        # question or exclamation mark (¿/¡), which the model never learns
+        # to do without. Mirrors augment_initial_punct in
+        # prepare_tokenizer_treebank.py, applied per sentence in
+        # __getitem__ instead of by duplicating sentences at dataset-
+        # preparation time. Unlike augment_nopunct, this uses a flat
+        # default ratio (not an auto-detected one), matching the ratio
+        # parameter augment_initial_punct itself takes.
+        drop_initial_punct_arg = args.get('drop_initial_punct_prob', 0.20)
+        if self.eval or not drop_initial_punct_arg or drop_initial_punct_arg <= 0:
+            self.drop_initial_punct_eligible = False
+            self.drop_initial_punct_ratio = 0.0
+        else:
+            self.drop_initial_punct_eligible = any(record_starts_with_mark(record) for record in self.data)
+            self.drop_initial_punct_ratio = drop_initial_punct_arg if self.drop_initial_punct_eligible else 0.0
+
     def init_vocab(self, data):
         assert self.eval == False # for eval vocab must exist
         cutoff = self.args['word_cutoff'] if self.args.get('word_cutoff') is not None else DEFAULT_WORD_CUTOFF
@@ -250,6 +336,11 @@ def __getitem__(self, key):
                 # lemma) and non-ROOT-prepended fields (head/deprel/text)
                 # all simply lose their last entry
                 record = [field[:-1] for field in record]
+        if self.drop_initial_punct_ratio > 0 and record_can_drop_initial_mark(record):
+            if random.random() < self.drop_initial_punct_ratio:
+                # drop the leading ¿/¡ and renumber every remaining word's
+                # head position down by one -- see drop_initial_mark_from_record
+                record = drop_initial_mark_from_record(record)
         return record
 
     def __iter__(self):

stanza/models/parser.py

@@ -240,6 +240,7 @@ def build_argparse():
     #
     # One simple way to fix this is to train on some fraction of training data with punct.
     parser.add_argument('--augment_nopunct', type=float, default=None, help='Fraction of punct-ending sentences to dynamically present without the final punct, applied fresh each epoch rather than by duplicating sentences in the training set.  Default of None will aim for roughly 10%%')
+    parser.add_argument('--drop_initial_punct_prob', type=float, default=0.20, help='Probability to drop a leading inverted question or exclamation mark (¿/¡) from a sentence, for languages such as Spanish and Catalan where it should be optional')
 
     parser.add_argument('--wandb', action='store_true', help='Start a wandb session and write the results of training.  Only applies to training.  Use --wandb_name instead to specify a name')
     parser.add_argument('--wandb_name', default=None, help='Name of a wandb session to start when training.  Will default to the dataset short name')

stanza/models/pos/data.py

@@ -9,7 +9,7 @@
 from torch.nn.utils.rnn import pad_sequence
 
 from stanza.models.common.bert_embedding import filter_data, needs_length_filter
-from stanza.models.common.data import map_to_ids, get_long_tensor, get_float_tensor, sort_all
+from stanza.models.common.data import map_to_ids, get_long_tensor, get_float_tensor, sort_all, starts_with_initial_mark
 from stanza.models.common.utils import DEFAULT_WORD_CUTOFF, simplify_punct
 from stanza.models.common.vocab import PAD_ID, VOCAB_PREFIX, CharVocab
 from stanza.models.pos.vocab import WordVocab, XPOSVocab, FeatureVocab, MultiVocab
@@ -54,6 +54,27 @@ def __init__(self, doc, args, pretrain, vocab=None, evaluation=False, sort_durin
             data = random.sample(data, keep)
             logger.debug("Subsample training set with rate {:g}".format(args['sample_train']))
 
+        # dynamic leading-¿/¡ drop: some UD treebanks (Spanish, Catalan) have
+        # every training sentence with a leading inverted question or
+        # exclamation mark, which the model never learns to do without.
+        #
+        # Eligibility is checked against the raw sentence data (before
+        # self.vocab['word'] is even queried), not by asking whether ¿/¡
+        # is IN self.vocab['word']. self.vocab['word'] is built with a
+        # frequency cutoff (DEFAULT_WORD_CUTOFF, or word_cutoff if set) --
+        # a word appearing fewer times than the cutoff is left out of the
+        # vocab and maps to UNK instead. In a small treebank, ¿/¡ could
+        # easily appear only a handful of times and fall under that
+        # cutoff, which would make a vocab-containment check wrongly say
+        # "ineligible" even though the mark is genuinely present in the
+        # data. Scanning the raw sentences directly avoids that failure
+        # mode, so the augmentation still triggers correctly even on
+        # small treebanks. starts_with_initial_mark is shared with the
+        # dependency parser's equivalent eligibility check.
+        self.drop_initial_punct_eligible = not self.eval and any(
+            starts_with_initial_mark([w[0] for w in sent]) for sent in data)
+        self.drop_initial_punct_ratio = args.get('drop_initial_punct_prob', 0.20) if self.drop_initial_punct_eligible else 0.0
+
         data = self.preprocess(data, self.vocab, self.pretrain_vocab, args)
 
         self.data = data
@@ -208,6 +229,25 @@ def __getitem__(self, key):
                 char = char[:mask] + char[mask+1:]
                 raw_text = raw_text[:mask] + raw_text[mask+1:]
 
+        # dynamic leading-¿/¡ drop (see drop_initial_punct_eligible in
+        # __init__). Unlike the trailing-punct mask above, this can't be
+        # done by masking a position in place: removing the FIRST word
+        # has to shift every later position back by one, so every field
+        # is sliced consistently rather than one element being replaced
+        # with a placeholder while the rest stay put.
+        if (self.drop_initial_punct_ratio > 0 and starts_with_initial_mark(raw_text)
+                and random.uniform(0, 1) < self.drop_initial_punct_ratio):
+            words = words[1:]
+            if upos is not None:
+                upos = upos[1:]
+            if xpos is not None:
+                xpos = xpos[1:]
+            if ufeats is not None:
+                ufeats = ufeats[1:]
+            pretrained = pretrained[1:]
+            char = char[1:]
+            raw_text = raw_text[1:]
+
         # get each character from the input sentnece
         # chars = [w for sent in char for w in sent]
 

stanza/models/tagger.py

@@ -117,6 +117,7 @@ def build_argparse():
     utils.add_device_args(parser)
 
     parser.add_argument('--augment_nopunct', type=float, default=None, help='Augment the training data by copying this fraction of punct-ending sentences as non-punct.  Default of None will aim for roughly 50%%')
+    parser.add_argument('--drop_initial_punct_prob', type=float, default=0.20, help='Probability to drop a leading inverted question or exclamation mark (¿/¡) from a sentence, for languages such as Spanish and Catalan where it should be optional')
 
     parser.add_argument('--wandb', action='store_true', help='Start a wandb session and write the results of training.  Only applies to training.  Use --wandb_name instead to specify a name')
     parser.add_argument('--wandb_name', default=None, help='Name of a wandb session to start when training.  Will default to the dataset short name')