Merge remote-tracking branch 'origin/dev' into corefud_v1.3

Jemoka · Jemoka · commit e1f56aaf13a8 · 2025-08-12T23:44:15.000-07:00
diff --git a/setup.py b/setup.py
@@ -64,6 +64,7 @@
         'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: 3.11',
         'Programming Language :: Python :: 3.12',
+        'Programming Language :: Python :: 3.13',
     ],
 
     # What does your project relate to?
diff --git a/stanza/models/depparse/scorer.py b/stanza/models/depparse/scorer.py
@@ -9,7 +9,7 @@
 
 logger = logging.getLogger('stanza')
 
-def score_named_dependencies(pred_doc, gold_doc):
+def score_named_dependencies(pred_doc, gold_doc, output_latex=False):
     if len(pred_doc.sentences) != len(gold_doc.sentences):
         logger.warning("Not evaluating individual dependency F1 on accound of document length mismatch")
         return
@@ -32,7 +32,13 @@ def score_named_dependencies(pred_doc, gold_doc):
     labels = sorted(set(tp.keys()).union(fp.keys()).union(fn.keys()))
     max_len = max(len(x) for x in labels)
     log_lines = []
-    log_line_fmt = "%" + str(max_len) + "s: p %.4f r %.4f f1 %.4f (%d actual)"
+    #log_line_fmt = "%" + str(max_len) + "s: p %.4f r %.4f f1 %.4f (%d actual)"
+    if output_latex:
+        log_lines.append(r"\begin{tabular}{lrr}")
+        log_lines.append(r"Reln & F1 & Total \\")
+        log_line_fmt = "{label} & {f1:0.4f} & {actual} \\\\"
+    else:
+        log_line_fmt = "{label:>" + str(max_len) + "s}: p {precision:0.4f} r {recall:0.4f} f1 {f1:0.4f} ({actual} actual)"
     for label in labels:
         if tp[label] == 0:
             precision = 0
@@ -42,7 +48,17 @@ def score_named_dependencies(pred_doc, gold_doc):
             precision = tp[label] / (tp[label] + fp[label])
             recall = tp[label] / (tp[label] + fn[label])
             f1 = 2 * (precision * recall) / (precision + recall)
-        log_lines.append(log_line_fmt % (label, precision, recall, f1, tp[label] + fn[label]))
+        actual = tp[label] + fn[label]
+        template = {
+            'label': label,
+            'precision': precision,
+            'recall': recall,
+            'f1': f1,
+            'actual': actual
+        }
+        log_lines.append(log_line_fmt.format(**template))
+    if output_latex:
+        log_lines.append(r"\end{tabular}")
     logger.info("F1 scores for each dependency:\n  Note that unlabeled attachment errors hurt the labeled attachment scores\n%s" % "\n".join(log_lines))
 
 def score(system_conllu_file, gold_conllu_file, verbose=True):
diff --git a/stanza/models/parser.py b/stanza/models/parser.py
@@ -47,6 +47,7 @@ def build_argparse():
     parser.add_argument('--eval_file', type=str, default=None, help='Input file for data loader.')
     parser.add_argument('--output_file', type=str, default=None, help='Output CoNLL-U file.')
     parser.add_argument('--no_gold_labels', dest='gold_labels', action='store_false', help="Don't score the eval file - perhaps it has no gold labels, for example.  Cannot be used at training time")
+    parser.add_argument('--output_latex', default=False, action='store_true', help='Output the per-relation table in Latex form')
     parser.add_argument('--mode', default='train', choices=['train', 'predict'])
     parser.add_argument('--lang', type=str, help='Language')
     parser.add_argument('--shorthand', type=str, help="Treebank shorthand")
@@ -422,7 +423,7 @@ def evaluate_trainer(args, trainer, pretrain):
                 if word.deprel is None:
                     raise ValueError("Gold document {} has a None at sentence {} word {}\n{:C}".format(args['eval_file'], sent_idx, word_idx, sentence))
 
-        scorer.score_named_dependencies(batch.doc, gold_doc)
+        scorer.score_named_dependencies(batch.doc, gold_doc, args['output_latex'])
         _, _, score = scorer.score(system_pred_file, args['eval_file'])
 
         logger.info("Parser score:")
diff --git a/stanza/pipeline/demo/Astloch-Bold.ttf b/stanza/pipeline/demo/Astloch-Bold.ttf
diff --git a/stanza/pipeline/demo/Liberation_Sans-Regular.ttf b/stanza/pipeline/demo/Liberation_Sans-Regular.ttf
diff --git a/stanza/pipeline/demo/PT_Sans-Caption-Web-Regular.ttf b/stanza/pipeline/demo/PT_Sans-Caption-Web-Regular.ttf
diff --git a/stanza/pipeline/demo/demo_server.py b/stanza/pipeline/demo/demo_server.py
@@ -12,9 +12,11 @@ def get_file(path):
     return res
 
 @app.route('/<path:path>')
+@app.route('/static/fonts/<path:path>')
 def static_file(path):
     if path in ['stanza-brat.css', 'stanza-brat.js', 'stanza-parseviewer.js', 'loading.gif',
-            'favicon.png', 'stanza-logo.png']:
+                'favicon.png', 'stanza-logo.png',
+                'Astloch-Bold.ttf', 'Liberation_Sans-Regular.ttf', 'PT_Sans-Caption-Web-Regular.ttf']:
         return app.send_static_file(path)
     elif path in 'index.html':
         return app.send_static_file('stanza-brat.html')
diff --git a/stanza/pipeline/demo/stanza-brat.html b/stanza/pipeline/demo/stanza-brat.html
@@ -1,5 +1,5 @@
 <html>
-<head profile="http://www.w3.org/2005/10/profile">
+<head profile="https://www.w3.org/2005/10/profile">
   <link rel='icon' href='favicon.png' type='image/png'/ >
   <!-- JQuery -->
   <script src="https://code.jquery.com/jquery-2.1.4.min.js"></script>
@@ -15,7 +15,7 @@
   <script type="text/javascript" src="https://nlp.stanford.edu/js/brat/client/lib/head.load.min.js"></script>
   <!-- d3 -->
   <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.17/d3.min.js"></script>
-  <script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/dagre-d3/0.4.17/dagre-d3.min.js"></script>
+  <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/dagre-d3/0.4.17/dagre-d3.min.js"></script>
 
     <!-- CoreNLP -->
   <link rel="stylesheet" type="text/css" href="stanza-brat.css"/>
diff --git a/stanza/resources/prepare_resources.py b/stanza/resources/prepare_resources.py
@@ -421,6 +421,11 @@ def get_default_processors(resources, lang):
         expected_lemma = default_package + "_nocharlm"
         if expected_lemma in resources[lang]['lemma']:
             default_processors['lemma'] = expected_lemma
+        else:
+            expected_lemma = default_package + "_charlm"
+            if expected_lemma in resources[lang]['lemma']:
+                default_processors['lemma'] = expected_lemma
+                print("WARNING: nocharlm lemmatizer for %s model does not exist, but %s does" % (default_package, expected_lemma))
     elif lang not in allowed_empty_languages:
         default_processors['lemma'] = 'identity'
 
@@ -603,6 +608,11 @@ def process_packages(args):
                     lemma_package = package + "_nocharlm"
                     if lemma_package in resources[lang]["lemma"]:
                         processors["lemma"] = lemma_package
+                    else:
+                        lemma_package = package + "_charlm"
+                        if lemma_package in resources[lang]['lemma']:
+                            processors['lemma'] = lemma_package
+                            print("WARNING: nocharlm lemmatizer for %s model does not exist, but %s does" % (package, lemma_package))
 
                 if "depparse" in resources[lang] and "pos" in processors:
                     depparse_package = None
diff --git a/stanza/server/semgrex.py b/stanza/server/semgrex.py
@@ -115,6 +115,7 @@ def annotate_doc(doc, semgrex_result, semgrex_patterns, matches_only, exclude_ma
                     sentence.add_comment("# semgrex pattern |%s| did not match!" % semgrex_pattern)
             else:
                 sentence_matched = True
+                highlight_tokens = []
                 for match in pattern_result.match:
                     match_word = "%d:%s" % (match.matchIndex, sentence.words[match.matchIndex-1].text)
                     if len(match.node) == 0:
@@ -124,7 +125,8 @@ def annotate_doc(doc, semgrex_result, semgrex_patterns, matches_only, exclude_ma
                                         for node in match.node]
                         node_matches = "  " + " ".join(node_matches)
                     sentence.add_comment("# semgrex pattern |%s| matched at %s%s" % (semgrex_pattern, match_word, node_matches))
-                    sentence.add_comment("# highlight tokens = %d" % match.matchIndex)
+                    highlight_tokens.append(match.matchIndex)
+                sentence.add_comment("# highlight tokens = %s" % (" ".join("%d" % x for x in highlight_tokens)))
         if sentence_matched:
             matching_sentences.append(sentence)
         else:
diff --git a/stanza/utils/datasets/ner/prepare_ner_dataset.py b/stanza/utils/datasets/ner/prepare_ner_dataset.py
@@ -79,6 +79,19 @@
     git clone git@github.com:ltrc/IL-NER.git
     python3 -m stanza.utils.datasets.ner.prepare_ner_dataset or_ilner
 
+suralk/multiNER contains three languages, EN, SI, and TA
+  https://github.com/suralk/multiNER
+  https://arxiv.org/abs/2412.02056
+  - Ranathunga, Surangika, et al.
+    A Multi-way Parallel Named Entity Annotated Corpus for English, Tamil and Sinhala
+  The tags are in BIO format, with the same 4 tags as CoNLL
+  Convert the data as follows:
+    cd $NERBASE
+    mkdir mixed
+    cd mixed
+    git clone git@github.com:suralk/multiNER.git
+    python3 -m stanza.utils.datasets.ner.prepare_ner_dataset ta_suralk
+
 Ukranian NER is provided by lang-uk, available here:
   https://github.com/lang-uk/ner-uk
   git clone the repo to $NERBASE/lang-uk
@@ -473,7 +486,7 @@
 
 from stanza.utils.datasets.common import UnknownDatasetError
 from stanza.utils.datasets.ner.preprocess_wikiner import preprocess_wikiner
-from stanza.utils.datasets.ner.split_wikiner import split_wikiner
+from stanza.utils.datasets.ner.split_wikiner import split_wikiner, split_wikiner_data
 import stanza.utils.datasets.ner.build_en_combined as build_en_combined
 import stanza.utils.datasets.ner.conll_to_iob as conll_to_iob
 import stanza.utils.datasets.ner.convert_ar_aqmar as convert_ar_aqmar
@@ -526,6 +539,25 @@ def process_it_fbk(paths, short_name):
     split_wikiner(base_output_path, csv_file, prefix=short_name, suffix="io", shuffle=False, train_fraction=0.8, dev_fraction=0.1)
     convert_bio_to_json(base_output_path, base_output_path, short_name, suffix="io")
 
+def process_suralk_multiner(paths, short_name):
+    lang_filenames = {
+        "en": "Final_English.txt",
+        "si": "Final_Sinhala.txt",
+        "ta": "Final_Tamil.txt",
+    }
+    lang, ending = short_name.split("_")
+    assert ending == "suralk"
+    assert lang in lang_filenames, "suralk/multiNER only supports %s" % (", ".join(lang_filenames.keys()))
+    suralk_path = os.path.join(paths["NERBASE"], "mixed", "multiNER", "nerannotateddatasets.zip")
+    if not os.path.exists(suralk_path):
+        raise FileNotFoundError("Expected to find the suralk/multiNER dataset in %s" % suralk_path)
+    sentences = read_tsv(lang_filenames[lang], text_column=0, annotation_column=1, separator=None, zip_filename=suralk_path)
+    print("Read %d sentences from %s::%s" % (len(sentences), suralk_path, lang_filenames[lang]))
+
+    base_output_path = paths["NER_DATA_DIR"]
+    split_wikiner_data(base_output_path, sentences, prefix=short_name, suffix="bio", shuffle=True)
+    convert_bio_to_json(base_output_path, base_output_path, short_name, suffix="bio")
+
 def process_il_ner(paths, short_name):
     joiner = chr(0x200c)
     def fix_tag(tag):
@@ -1481,6 +1513,7 @@ def process_he_iahlt(paths, short_name):
 
 SUFFIX_MAPPING = {
     "_ilner":            process_il_ner,
+    "_suralk":           process_suralk_multiner,
 }
 
 def main(dataset_name):
diff --git a/stanza/utils/datasets/prepare_tokenizer_treebank.py b/stanza/utils/datasets/prepare_tokenizer_treebank.py
@@ -993,6 +993,37 @@ def strip_feats(sents):
     """
     return strip_column(sents, 5)
 
+def build_combined_japanese_dataset(paths, model_type, dataset):
+    """
+    GSD with a handparsed dataset of some short verb phrases
+    """
+    udbase_dir = paths["UDBASE"]
+    handparsed_dir = paths["HANDPARSED_DIR"]
+
+    treebank = "UD_Japanese-GSD"
+    conllu_file = common.find_treebank_dataset_file(treebank, udbase_dir, dataset, "conllu", fail=True)
+    gsd_sents = read_sentences_from_conllu(conllu_file)
+    print("Read %d sentences from %s" % (len(gsd_sents), conllu_file))
+
+    if dataset == 'train':
+        extra_japanese = os.path.join(handparsed_dir, "japanese-handparsed", "spaces-ready-checked.conllu")
+        if not os.path.exists(extra_japanese):
+            raise FileNotFoundError("Cannot find the extra dataset which includes various verb patterns, expected {}".format(extra_japanese))
+        extra_sents = read_sentences_from_conllu(extra_japanese)
+        print("Read %d sentences from %s" % (len(extra_sents), extra_japanese))
+
+        if model_type == common.ModelType.POS:
+            documents = {}
+            documents[treebank] = gsd_sents
+            documents['handparsed'] = extra_sents
+            return documents
+        else:
+            sents = gsd_sents + extra_sents
+            return sents
+    else:
+        return gsd_sents
+
+
 def build_combined_albanian_dataset(paths, model_type, dataset):
     """
     sq_combined is STAF as the base, with TSA added for some things
@@ -1099,7 +1130,7 @@ def build_combined_spanish_dataset(paths, model_type, dataset):
         if model_type in (common.ModelType.TOKENIZER, common.ModelType.MWT, common.ModelType.LEMMA):
             extra_spanish = os.path.join(handparsed_dir, "spanish-mwt", "adjectives.conllu")
             if not os.path.exists(extra_spanish):
-                raise FileNotFoundError("Cannot find the extra dataset 'handpicked.mwt' which includes various multi-words retokenized, expected {}".format(extra_italian))
+                raise FileNotFoundError("Cannot find the extra dataset 'adjectives.conllu' which includes various multi-words retokenized, expected {}".format(extra_spanish))
             extra_sents = read_sentences_from_conllu(extra_spanish)
             print("Read %d sentences from %s" % (len(extra_sents), extra_spanish))
             sents.extend(extra_sents)
@@ -1177,6 +1208,7 @@ def build_combined_hebrew_dataset(paths, model_type, dataset):
     "fr_combined": build_combined_french_dataset,
     "he_combined": build_combined_hebrew_dataset,
     "it_combined": build_combined_italian_dataset,
+    "ja_combined": build_combined_japanese_dataset,
     "sq_combined": build_combined_albanian_dataset,
 }
 
diff --git a/stanza/utils/training/run_lemma.py b/stanza/utils/training/run_lemma.py
@@ -20,11 +20,9 @@
 
 from stanza.models import identity_lemmatizer
 from stanza.models import lemmatizer
-from stanza.models.lemma import attach_lemma_classifier
 
 from stanza.utils.training import common
 from stanza.utils.training.common import Mode, add_charlm_args, build_lemma_charlm_args, choose_lemma_charlm
-from stanza.utils.training import run_lemma_classifier
 
 from stanza.utils.datasets.prepare_lemma_treebank import check_lemmas
 import stanza.utils.datasets.prepare_lemma_classifier as prepare_lemma_classifier
@@ -148,6 +146,12 @@ def run_treebank(mode, paths, treebank, short_name,
             use_lemma_classifier = command_args.charlm is not None
         use_lemma_classifier = use_lemma_classifier and short_name in prepare_lemma_classifier.DATASET_MAPPING
         if use_lemma_classifier and mode == Mode.TRAIN:
+            # some installations may not have transformers,
+            # so we bury the lemma_classifier import in the codepath
+            # which actually needs it
+            from stanza.models.lemma import attach_lemma_classifier
+            from stanza.utils.training import run_lemma_classifier
+
             lc_charlm_args = ['--no_charlm'] if command_args.charlm is None else ['--charlm', command_args.charlm]
             lemma_classifier_args = [treebank] + lc_charlm_args
             if command_args.force:
