Update llama-vocab.cpp

Author: danielhanchen

src/llama-vocab.cpp

@@ -431,18 +431,16 @@ struct llm_tokenizer_bpe : llm_tokenizer {
                     // 1. Han characters
                     "[\\p{Han}]+",
                     // 2. Words ending in lowercase (non-Han) with contractions
-                    "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?",
-                    // 3. Words starting with uppercase (non-Han) with contractions
-                    "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?",
-                    // 4. Numbers (1-3 digits)
+                    "[^\\r\\n\\p{L}\\p{N}]?(?:(?!\\p{Han})(?:\\p{Lu}|\\p{Lt}|\\p{Lm}|\\p{Lo}|\\p{M})*(?!\\p{Han})(?:\\p{Ll}|\\p{Lm}|\\p{Lo}|\\p{M})+|(?!\\p{Han})(?:\\p{Lu}|\\p{Lt}|\\p{Lm}|\\p{Lo}|\\p{M})+(?!\\p{Han})(?:\\p{Ll}|\\p{Lm}|\\p{Lo}|\\p{M})*)(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?",
+                    // 3. Numbers (1-3 digits)
                     "\\p{N}{1,3}",
-                    // 5. Punctuation and symbols
+                    // 4. Punctuation and symbols
                     " ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*",
-                    // 6. Newlines
+                    // 5. Newlines
                     "\\s*[\\r\\n]+",
-                    // 7.Whitespace at the end of a line
+                    // 6.Whitespace at the end of a line
                     "\\s+(?!\\S)",
-                    // 8. General whitespace
+                    // 7. General whitespace
                     "\\s+",
                 };
                 break;