diff --git a/CMakeLists.txt b/CMakeLists.txt index 2dbb45effe..afb8e04c72 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,7 +30,7 @@ if(UNIX AND NOT (APPLE OR ANDROID OR CYGWIN)) endif() project(OpenVINOGenAI - VERSION 2026.3.0.0 + VERSION 2026.4.0.0 DESCRIPTION "OpenVINO GenAI" HOMEPAGE_URL "https://github.com/openvinotoolkit/openvino.genai" LANGUAGES CXX C) diff --git a/pyproject.toml b/pyproject.toml index 0a1cf8222a..bb93c5333d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "openvino-genai" -version = "2026.3.0.0" +version = "2026.4.0.0" description = "Library of the most popular Generative AI model pipelines, optimized execution methods, and samples" requires-python = ">=3.10" readme = { file = "src/README.md", content-type="text/markdown" } @@ -29,7 +29,7 @@ classifiers = [ "Programming Language :: Python :: Implementation :: CPython" ] dependencies = [ - "openvino_tokenizers~=2026.3.0.0.dev" + "openvino_tokenizers~=2026.4.0.0.dev" ] [project.optional-dependencies] testing = ["pytest>=6.0"] @@ -53,7 +53,7 @@ options = {"ENABLE_PYTHON" = "ON", "BUILD_TOKENIZERS" = "OFF", "ENABLE_SAMPLES" [build-system] requires = [ "py-build-cmake==0.5.0", - "openvino~=2026.3.0.0.dev", + "openvino~=2026.4.0.0.dev", "pybind11-stubgen==2.5.5", "cmake~=3.23.0; platform_system != 'Darwin' or platform_machine == 'x86_64'", "cmake~=4.3.0; platform_system == 'Darwin' and platform_machine == 'arm64'", diff --git a/samples/deployment-requirements.txt b/samples/deployment-requirements.txt index 3503f28276..8ef87b0f97 100644 --- a/samples/deployment-requirements.txt +++ b/samples/deployment-requirements.txt @@ -1,5 +1,5 @@ --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly -openvino_genai~=2026.3.0.0.dev +openvino_genai~=2026.4.0.0.dev librosa==0.11.0 # For Whisper pillow==12.3.0 # Image processing for VLMs json5==0.15.0 # For ReAct diff --git a/samples/export-requirements.txt b/samples/export-requirements.txt index cf02241ae0..fc7740fb37 100644 --- a/samples/export-requirements.txt +++ b/samples/export-requirements.txt @@ -1,6 +1,6 @@ --extra-index-url https://download.pytorch.org/whl/cpu --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly -openvino-tokenizers[transformers]~=2026.3.0.0.dev +openvino-tokenizers[transformers]~=2026.4.0.0.dev https://github.com/huggingface/optimum-intel/archive/a8c4734741e766ef95d7f1a7d1e29a1d4ba2ab8f.tar.gz#egg=optimum-intel einops==0.8.2 # For Qwen transformers_stream_generator==0.0.5 # For Qwen diff --git a/src/cpp/src/tokenizer/tokenizer_impl.cpp b/src/cpp/src/tokenizer/tokenizer_impl.cpp index a359cc064d..d6e4f1d0dc 100644 --- a/src/cpp/src/tokenizer/tokenizer_impl.cpp +++ b/src/cpp/src/tokenizer/tokenizer_impl.cpp @@ -787,6 +787,31 @@ std::vector Tokenizer::TokenizerImpl::decode(const std::vector(res_data, res_data + res.get_shape()[0]); } +std::shared_ptr Tokenizer::TokenizerImpl::get_cached_minja_chat_template(const std::string& chat_template) const { + constexpr size_t max_cached_templates = 4; + + { + std::lock_guard lock(m_minja_chat_template_cache_mutex); + auto iter = m_minja_chat_template_cache.find(chat_template); + if (iter != m_minja_chat_template_cache.end()) { + return iter->second; + } + } + + auto minja_template = std::make_shared(chat_template, m_bos_token, m_eos_token); + + std::lock_guard lock(m_minja_chat_template_cache_mutex); + auto iter = m_minja_chat_template_cache.find(chat_template); + if (iter != m_minja_chat_template_cache.end()) { + return iter->second; + } + if (m_minja_chat_template_cache.size() >= max_cached_templates) { + m_minja_chat_template_cache.clear(); + } + m_minja_chat_template_cache.emplace(chat_template, minja_template); + return minja_template; +} + std::string Tokenizer::TokenizerImpl::apply_chat_template( const ChatHistory& history, bool add_generation_prompt, @@ -794,7 +819,13 @@ std::string Tokenizer::TokenizerImpl::apply_chat_template( const std::optional& tools, const std::optional& extra_context ) const { - std::string chat_tpl = chat_template.empty() ? m_chat_template : remap_template(chat_template); + std::string chat_tpl; + if (chat_template.empty()) { + std::lock_guard lock(m_minja_chat_template_cache_mutex); + chat_tpl = m_chat_template; + } else { + chat_tpl = remap_template(chat_template); + } OPENVINO_ASSERT(!chat_tpl.empty(), "Chat template wasn't found. This may indicate that the model wasn't trained for chat scenario." " Please add 'chat_template' to tokenizer_config.json to use the model in chat scenario." @@ -808,7 +839,7 @@ std::string Tokenizer::TokenizerImpl::apply_chat_template( OPENVINO_ASSERT(resolved_extra_context.is_object(), "Extra context should be an object-like JsonContainer, got: ", resolved_extra_context.type_name()); - minja::chat_template minja_template(chat_tpl, m_bos_token, m_eos_token); + auto minja_template = get_cached_minja_chat_template(chat_tpl); minja::chat_template_inputs minja_inputs; minja_inputs.messages = history.get_messages(); @@ -826,7 +857,7 @@ std::string Tokenizer::TokenizerImpl::apply_chat_template( std::string result; try { - result = minja_template.apply(minja_inputs); + result = minja_template->apply(minja_inputs); } catch (const std::exception& error) { OPENVINO_THROW("Minja failed to apply chat template. Possible solutions are\n" "* Provide a simplified chat template with set_chat_template().\n" @@ -842,15 +873,21 @@ std::string Tokenizer::TokenizerImpl::apply_chat_template( } void Tokenizer::TokenizerImpl::set_chat_template(const std::string& chat_template) { + auto remapped_chat_template = remap_template(chat_template); + + std::lock_guard lock(m_minja_chat_template_cache_mutex); m_original_chat_template = chat_template; - m_chat_template = remap_template(chat_template); + m_chat_template = std::move(remapped_chat_template); + m_minja_chat_template_cache.clear(); } std::string Tokenizer::TokenizerImpl::get_chat_template() const { + std::lock_guard lock(m_minja_chat_template_cache_mutex); return m_chat_template; } std::string Tokenizer::TokenizerImpl::get_original_chat_template() const { + std::lock_guard lock(m_minja_chat_template_cache_mutex); return m_original_chat_template; } diff --git a/src/cpp/src/tokenizer/tokenizer_impl.hpp b/src/cpp/src/tokenizer/tokenizer_impl.hpp index 932be2c0a1..73fa9264b9 100644 --- a/src/cpp/src/tokenizer/tokenizer_impl.hpp +++ b/src/cpp/src/tokenizer/tokenizer_impl.hpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include "minja/minja.hpp" #include "minja/chat-template.hpp" @@ -43,6 +45,8 @@ class Tokenizer::TokenizerImpl { std::string m_eos_token = {}; std::string m_chat_template = {}; std::string m_original_chat_template = {}; + mutable std::mutex m_minja_chat_template_cache_mutex; + mutable std::unordered_map> m_minja_chat_template_cache; std::vector m_vocab = {}; std::shared_ptr m_structured_output_controller = nullptr; @@ -83,6 +87,9 @@ class Tokenizer::TokenizerImpl { std::string get_chat_template() const; std::string get_original_chat_template() const; std::shared_ptr get_structured_output_controller(std::optional vocab_size = std::nullopt); + +private: + std::shared_ptr get_cached_minja_chat_template(const std::string& chat_template) const; }; } // namespace genai diff --git a/src/js/package-lock.json b/src/js/package-lock.json index cc8a76dfe0..6e2c48840d 100644 --- a/src/js/package-lock.json +++ b/src/js/package-lock.json @@ -271,9 +271,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "24.13.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.2.tgz", - "integrity": "sha512-fRa09kZTgu8o71KFcDjUFuc7F+dEbZYZmkI0mg5YBTRs0yMKjYHsq/c0urDKeDb+D5qVgXOdFcuu+DZPKOITwA==", + "version": "24.13.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", + "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", "dev": true, "license": "MIT", "dependencies": { diff --git a/tests/python_tests/test_llm_pipeline_static.py b/tests/python_tests/test_llm_pipeline_static.py index 28eac66810..170b383370 100644 --- a/tests/python_tests/test_llm_pipeline_static.py +++ b/tests/python_tests/test_llm_pipeline_static.py @@ -332,6 +332,7 @@ def test_terminate_by_max_number_of_tokens( @pytest.mark.parametrize("llm_model", MODELS_LIST, indirect=True) @pytest.mark.parametrize("npu_config", PIPELINE_CONFIGS, indirect=True) +@pytest.mark.xfail(reason="Error: KV-Cache is full: num_stored_tokens=319 capacity=319. Ticket 190518") def test_terminate_by_out_of_memory( llm_model: OVConvertedModelSchema, npu_config: dict, diff --git a/thirdparty/openvino_tokenizers b/thirdparty/openvino_tokenizers index 2d2839dbcb..6caf6c1857 160000 --- a/thirdparty/openvino_tokenizers +++ b/thirdparty/openvino_tokenizers @@ -1 +1 @@ -Subproject commit 2d2839dbcb92ad9c5830d8c0d795d403bcdbeaad +Subproject commit 6caf6c185751069ecad6132fb595642b3daa2cb2