disable warning about tokenizers version for ov tokenizers >= 2024.5 (#962)

* disable warning about tokenizers version for ov tokenizers >= 2024.5

* increase verbosity for tokenizers test

Author: eaidova

optimum/exporters/openvino/__main__.py

@@ -55,6 +55,9 @@
 
 logger = logging.getLogger(__name__)
 
+# init core before import openvino tokenizers to prevent failed attempt loading extension
+core = Core()
+
 
 def infer_task(
     task,
@@ -413,7 +416,6 @@ class StoreAttr(object):
     del model
     gc.collect()
 
-    core = Core()
     for submodel_path in submodel_paths:
         submodel_path = Path(output) / submodel_path
         submodel = core.read_model(submodel_path)

optimum/exporters/openvino/convert.py

@@ -41,6 +41,7 @@
     _torch_version,
     _transformers_version,
     compare_versions,
+    is_openvino_tokenizers_version,
     is_tokenizers_version,
     is_transformers_version,
 )
@@ -734,9 +735,9 @@ def export_tokenizer(
     except ModuleNotFoundError:
         return
 
-    if is_tokenizers_version(">", "0.19"):
+    if is_tokenizers_version(">", "0.19") and is_openvino_tokenizers_version("<", "2024.5.0.0"):
         logger.warning(
-            "Exporting tokenizers to OpenVINO is not supported for tokenizers version > 0.19. "
+            "Exporting tokenizers to OpenVINO is not supported for tokenizers version > 0.19 and openvino version <= 2024.4. "
             "Please downgrade to tokenizers version <= 0.19 to export tokenizers to OpenVINO."
         )
 

optimum/intel/utils/import_utils.py

@@ -382,6 +382,24 @@ def is_openvino_version(operation: str, version: str):
     return compare_versions(parse(_openvino_version), operation, version)
 
 
+def is_openvino_tokenizers_version(operation: str, version: str):
+    if not is_openvino_available():
+        return False
+    if not is_openvino_tokenizers_available():
+        return False
+    import openvino_tokenizers
+
+    tokenizers_version = openvino_tokenizers.__version__
+
+    if tokenizers_version == "0.0.0.0":
+        try:
+            tokenizers_version = importlib_metadata.version("openvino_tokenizers") or tokenizers_version
+        except importlib_metadata.PackageNotFoundError:
+            pass
+
+    return compare_versions(parse(tokenizers_version), operation, version)
+
+
 def is_diffusers_version(operation: str, version: str):
     """
     Compare the current diffusers version to a given reference with an operation.

tests/openvino/test_exporters_cli.py

@@ -153,7 +153,7 @@ def test_exporters_cli(self, task: str, model_type: str):
     def test_exporters_cli_tokenizers(self, task: str, model_type: str):
         with TemporaryDirectory() as tmpdir:
             output = subprocess.check_output(
-                f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdir}",
+                f"TRANSFORMERS_VERBOSITY=debug optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdir}",
                 shell=True,
                 stderr=subprocess.STDOUT,
             ).decode()