Expose Python function to check if a language code is valid (#311)

Author: guillaumekln

bindings/python/README.md

@@ -363,4 +363,7 @@ pyonmttok.is_placeholder(token: str)
 
 # Sets the random seed for reproducible tokenization.
 pyonmttok.set_random_seed(seed: int)
+
+# Checks if the language code is valid.
+pyonmttok.is_valid_language(lang: str).
 ```

bindings/python/pyonmttok/Python.cc

@@ -495,6 +495,7 @@ static onmt::Vocab create_vocab(const std::optional<std::vector<std::string>>& s
 PYBIND11_MODULE(_ext, m)
 {
   m.def("is_placeholder", &onmt::Tokenizer::is_placeholder, py::arg("token"));
+  m.def("is_valid_language", &onmt::unicode::is_valid_language, py::arg("lang"));
   m.def("set_random_seed", &onmt::set_random_seed, py::arg("seed"));
 
   py::enum_<onmt::Casing>(m, "Casing")

bindings/python/pyonmttok/__init__.py

@@ -29,6 +29,7 @@
     TokenType,
     Vocab,
     is_placeholder,
+    is_valid_language,
     set_random_seed,
 )
 from pyonmttok.version import __version__

bindings/python/test/test.py

@@ -17,6 +17,12 @@ def test_is_placeholder():
     assert pyonmttok.is_placeholder("｟hello｠")
 
 
+def test_is_valid_language():
+    assert pyonmttok.is_valid_language("fr")
+    assert pyonmttok.is_valid_language("de")
+    assert not pyonmttok.is_valid_language("xx")
+
+
 def test_simple():
     tokenizer = pyonmttok.Tokenizer("aggressive", joiner_annotate=True, joiner_new=True)
     text = "Hello World!"