You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
Copy file name to clipboardExpand all lines: stanza/models/tokenizer.py
+3-3Lines changed: 3 additions & 3 deletions
Original file line number
Diff line number
Diff line change
@@ -74,9 +74,9 @@ def build_argparse():
74
74
parser.add_argument('--last_char_drop_prob', type=float, default=0.2, help="Probability to drop the last char of a block of text during training, uniformly at random. Idea is to fake a document ending w/o sentence final punctuation, hopefully to avoid the tokenizer learning to always tokenize the last character as a period")
75
75
parser.add_argument('--last_char_move_prob', type=float, default=0.02, help="Probability to move the sentence final punctuation of a sentence during training, uniformly at random. Idea is to teach the tokenizer that a space separated sentence final punct still ends the sentence")
76
76
parser.add_argument('--punct_move_back_prob', type=float, default=0.02, help="Probability to move a comma in the sentence one over, removing the previous space, during training. Idea is to teach the tokenizer that commas can appear next to words even in languages where the dataset doesn't allow it, such as Vietnamese")
77
-
parser.add_argument('--split_mwt_prob', type=float, default=0.01, help="Probably to split an MWT into its component pieces and turn it into separate words")
78
-
parser.add_argument('--comma_typo_prob', type=float, default=0.01, help="Probably to fake a comma typo - eg 'foo, bar' -> 'foo ,bar'")
79
-
parser.add_argument('--comma_glue_prob', type=float, default=0.01, help="Probably to fake a comma gluing - eg 'foo, bar' -> 'foo,bar'")
77
+
parser.add_argument('--split_mwt_prob', type=float, default=0.01, help="Probability to split an MWT into its component pieces and turn it into separate words")
78
+
parser.add_argument('--comma_typo_prob', type=float, default=0.01, help="Probability to fake a comma typo - eg 'foo, bar' -> 'foo ,bar'")
79
+
parser.add_argument('--comma_glue_prob', type=float, default=0.01, help="Probability to fake a comma gluing - eg 'foo, bar' -> 'foo,bar'")
80
80
parser.add_argument('--augment_final_punct_prob', type=float, default=0.02, help="Probability to replace a ? with a ? or other similar augmentations")
81
81
parser.add_argument('--augment_mid_punct_prob', type=float, default=0.02, help="Probability to replace a , with a – or other similar augmentations")
0 commit comments