Merge branch 'master' into generate-charsmaps-during-build

Author: apaniukov

.github/workflows/mac.yml

@@ -1,4 +1,4 @@
-name: macOS (13, Python 3.11)
+name: macOS ARM64 (mac-14, Python 3.11)
 on:
   workflow_dispatch:
   pull_request:
@@ -20,7 +20,7 @@ env:
   CMAKE_CXX_COMPILER_LAUNCHER: ccache
   CMAKE_C_COMPILER_LAUNCHER: ccache
   CCACHE_MAXSIZE: 100Mi
-  BASE_PRODUCT_TYPE: public_darwin
+  BASE_PRODUCT_TYPE: public_macos_arm64
 
 permissions: read-all
 
@@ -46,7 +46,8 @@ jobs:
     - uses: openvinotoolkit/openvino/.github/actions/openvino_provider@master
       id: openvino_download
       with:
-        platform: 'macos_12_6'
+        platform: macos_14_7
+        arch: 'arm64'
         commit_packages_to_provide: 'wheels,developer_package.tar.gz'
         revision: latest_available_commit
 
@@ -60,7 +61,7 @@ jobs:
     defaults:
       run:
         shell: bash
-    runs-on: macos-13
+    runs-on: macos-14
 
     env:
       CMAKE_GENERATOR: 'Ninja'
@@ -77,14 +78,21 @@ jobs:
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           path: ${{ env.OPENVINO_TOKENIZERS_REPO }}
-          
+
+      - name: Setup Python ${{ env.PYTHON_VERSION }}
+        uses: openvinotoolkit/openvino/.github/actions/setup_python@master
+        with:
+          version: ${{ env.PYTHON_VERSION }}
+          should-setup-pip-paths: 'false'
+          self-hosted-runner: 'false'
+
       - name: Generate product manifest
         id: create_manifest
         uses: openvinotoolkit/openvino/.github/actions/create_manifest@master
         with:
           repos: ${{ env.OPENVINO_TOKENIZERS_REPO }}
           product_type: ${{ env.BASE_PRODUCT_TYPE }}_${{ matrix.build_type }}
-          target_arch: 'x86_64'
+          target_arch: 'arm64'
           build_type: ${{ matrix.build_type }}
           save_to: ${{ env.MANIFEST_PATH }}
 
@@ -167,7 +175,7 @@ jobs:
     defaults:
       run:
         shell: bash
-    runs-on: macos-13
+    runs-on: macos-14
 
     env:
       OPENVINO_REPO: ${{ github.workspace }}/openvino
@@ -204,7 +212,7 @@ jobs:
         with:
           repos: ${{ env.OPENVINO_TOKENIZERS_REPO }}
           product_type: ${{ env.BASE_PRODUCT_TYPE }}_release
-          target_arch: 'x86_64'
+          target_arch: 'arm64'
           build_type: Release
           save_to: ${{ env.MANIFEST_PATH }}
 
@@ -243,7 +251,6 @@ jobs:
           python3 -m pip wheel -v \
             --no-deps --wheel-dir ${BUILD_DIR} \
             --config-settings='override=wheel.build_tag="${{ github.run_number }}"' \
-            --config-settings=override=cross.arch="macosx_10_12_x86_64" \
             ${{ needs.openvino_download.outputs.ov_wheel_source }} \
             ${OPENVINO_TOKENIZERS_REPO}
         working-directory: ${{ env.INSTALL_DIR }}
@@ -278,7 +285,7 @@ jobs:
     defaults:
       run:
         shell: bash
-    runs-on: macos-13
+    runs-on: macos-14
     env:
       INSTALL_DIR: ${{ github.workspace }}/openvino/install
 

CMakeLists.txt

@@ -42,7 +42,7 @@ else()
 endif()
 
 project(openvino_tokenizers
-        VERSION 2026.0.0.0
+        VERSION 2026.1.0.0
         DESCRIPTION "OpenVINO Tokenizers"
         HOMEPAGE_URL "https://github.com/openvinotoolkit/openvino_tokenizers"
         LANGUAGES CXX C)

benchmark/benchmark.py

@@ -28,7 +28,8 @@ def sample_texts(
     dataset = [data for data in dataset if len(data["conversations"]) >= 2]
     # Keep the first two turns of each conversation
     dataset = [
-        (data["conversations"][0]["value"], data["conversations"][1]["value"]) for data in sample(dataset, k=num_texts)
+        (data["conversations"][0]["value"], data["conversations"][1]["value"]) 
+        for data in sample(dataset, k=num_texts)  # nosec
     ]
     shuffle(dataset)
     return dataset

cmake/templates/__version__.py.in

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2018-2025 Intel Corporation
+# Copyright (C) 2018-2026 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
 __version__ = "@OpenVINOTokenizers_FULL_VERSION@"

cmake/version.cmake

@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2025 Intel Corporation
+# Copyright (C) 2018-2026 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 #
 

js/README.md

@@ -1,6 +1,6 @@
 [project]
 name = "openvino-tokenizers"
-version = "2026.0.0.0"
+version = "2026.1.0.0"
 description = "Convert tokenizers into OpenVINO models"
 requires-python = ">=3.10"
 readme = { file = "README.md", content-type="text/markdown" }
@@ -37,7 +37,7 @@ classifiers = [
 
 dependencies = [
     # support of nightly openvino packages with dev suffix
-    "openvino~=2026.0.0.dev"
+    "openvino~=2026.1.0.dev"
 ]
 
 [project.optional-dependencies]
@@ -98,7 +98,7 @@ python_abi = "none"
 requires = [
     "py-build-cmake==0.4.3",
     "cmake~=3.14",
-    "openvino~=2026.0.0.dev"
+    "openvino~=2026.1.0.dev"
 ]
 build-backend = "py_build_cmake.build"
 

pyproject.toml

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2018-2025 Intel Corporation
+# Copyright (C) 2018-2026 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 import functools
 import logging

python/openvino_tokenizers/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2018-2025 Intel Corporation
+# Copyright (C) 2018-2026 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
 import logging

python/openvino_tokenizers/convert_tokenizer.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2018-2022 Intel Corporation
+# Copyright (C) 2018-2026 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 import functools
 import json