[Release] Cache Minja Template

CMakeLists.txt

@@ -30,7 +30,7 @@ if(UNIX AND NOT (APPLE OR ANDROID OR CYGWIN))
 endif()
 
 project(OpenVINOGenAI
-        VERSION 2026.3.0.0
+        VERSION 2026.4.0.0
         DESCRIPTION "OpenVINO GenAI"
         HOMEPAGE_URL "https://github.com/openvinotoolkit/openvino.genai"
         LANGUAGES CXX C)

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "openvino-genai"
-version = "2026.3.0.0"
+version = "2026.4.0.0"
 description = "Library of the most popular Generative AI model pipelines, optimized execution methods, and samples"
 requires-python = ">=3.10"
 readme = { file = "src/README.md", content-type="text/markdown" }
@@ -29,7 +29,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython"
 ]
 dependencies = [
-    "openvino_tokenizers~=2026.3.0.0.dev"
+    "openvino_tokenizers~=2026.4.0.0.dev"
 ]
 [project.optional-dependencies]
 testing = ["pytest>=6.0"]
@@ -53,7 +53,7 @@ options = {"ENABLE_PYTHON" = "ON", "BUILD_TOKENIZERS" = "OFF", "ENABLE_SAMPLES"
 [build-system]
 requires = [
     "py-build-cmake==0.5.0",
-    "openvino~=2026.3.0.0.dev",
+    "openvino~=2026.4.0.0.dev",
     "pybind11-stubgen==2.5.5",
     "cmake~=3.23.0; platform_system != 'Darwin' or platform_machine == 'x86_64'",
     "cmake~=4.3.0; platform_system == 'Darwin' and platform_machine == 'arm64'",

samples/deployment-requirements.txt

@@ -1,5 +1,5 @@
 --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
-openvino_genai~=2026.3.0.0.dev
+openvino_genai~=2026.4.0.0.dev
 librosa==0.11.0  # For Whisper
 pillow==12.3.0  # Image processing for VLMs
 json5==0.15.0  # For ReAct

samples/export-requirements.txt

@@ -1,6 +1,6 @@
 --extra-index-url https://download.pytorch.org/whl/cpu
 --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
-openvino-tokenizers[transformers]~=2026.3.0.0.dev
+openvino-tokenizers[transformers]~=2026.4.0.0.dev
 https://github.com/huggingface/optimum-intel/archive/a8c4734741e766ef95d7f1a7d1e29a1d4ba2ab8f.tar.gz#egg=optimum-intel
 einops==0.8.2  # For Qwen
 transformers_stream_generator==0.0.5  # For Qwen

src/cpp/src/tokenizer/tokenizer_impl.cpp

@@ -787,14 +787,45 @@ std::vector<std::string> Tokenizer::TokenizerImpl::decode(const std::vector<std:
     return std::vector<std::string>(res_data, res_data + res.get_shape()[0]);
 }
 
+std::shared_ptr<const minja::chat_template> Tokenizer::TokenizerImpl::get_cached_minja_chat_template(const std::string& chat_template) const {
+    constexpr size_t max_cached_templates = 4;
+
+    {
+        std::lock_guard<std::mutex> lock(m_minja_chat_template_cache_mutex);
+        auto iter = m_minja_chat_template_cache.find(chat_template);
+        if (iter != m_minja_chat_template_cache.end()) {
+            return iter->second;
+        }
+    }
+
+    auto minja_template = std::make_shared<minja::chat_template>(chat_template, m_bos_token, m_eos_token);
+
+    std::lock_guard<std::mutex> lock(m_minja_chat_template_cache_mutex);
+    auto iter = m_minja_chat_template_cache.find(chat_template);
+    if (iter != m_minja_chat_template_cache.end()) {
+        return iter->second;
+    }
+    if (m_minja_chat_template_cache.size() >= max_cached_templates) {
+        m_minja_chat_template_cache.clear();
+    }
+    m_minja_chat_template_cache.emplace(chat_template, minja_template);
+    return minja_template;
+}
+
 std::string Tokenizer::TokenizerImpl::apply_chat_template(
     const ChatHistory& history,
     bool add_generation_prompt,
     const std::string& chat_template,
     const std::optional<JsonContainer>& tools,
     const std::optional<JsonContainer>& extra_context
 ) const {
-    std::string chat_tpl = chat_template.empty() ? m_chat_template : remap_template(chat_template);
+    std::string chat_tpl;
+    if (chat_template.empty()) {
+        std::lock_guard<std::mutex> lock(m_minja_chat_template_cache_mutex);
+        chat_tpl = m_chat_template;
+    } else {
+        chat_tpl = remap_template(chat_template);
+    }
     OPENVINO_ASSERT(!chat_tpl.empty(),
                     "Chat template wasn't found. This may indicate that the model wasn't trained for chat scenario."
                     " Please add 'chat_template' to tokenizer_config.json to use the model in chat scenario."
@@ -808,7 +839,7 @@ std::string Tokenizer::TokenizerImpl::apply_chat_template(
     OPENVINO_ASSERT(resolved_extra_context.is_object(),
                     "Extra context should be an object-like JsonContainer, got: ", resolved_extra_context.type_name());
 
-    minja::chat_template minja_template(chat_tpl, m_bos_token, m_eos_token);
+    auto minja_template = get_cached_minja_chat_template(chat_tpl);
 
     minja::chat_template_inputs minja_inputs;
     minja_inputs.messages = history.get_messages();
@@ -826,7 +857,7 @@ std::string Tokenizer::TokenizerImpl::apply_chat_template(
 
     std::string result;
     try {
-        result = minja_template.apply(minja_inputs);
+        result = minja_template->apply(minja_inputs);
     } catch (const std::exception& error) {
         OPENVINO_THROW("Minja failed to apply chat template. Possible solutions are\n"
                         "* Provide a simplified chat template with set_chat_template().\n"
@@ -842,15 +873,21 @@ std::string Tokenizer::TokenizerImpl::apply_chat_template(
 }
 
 void Tokenizer::TokenizerImpl::set_chat_template(const std::string& chat_template) {
+    auto remapped_chat_template = remap_template(chat_template);
+
+    std::lock_guard<std::mutex> lock(m_minja_chat_template_cache_mutex);
     m_original_chat_template = chat_template;
-    m_chat_template = remap_template(chat_template);
+    m_chat_template = std::move(remapped_chat_template);
+    m_minja_chat_template_cache.clear();
 }
 
 std::string Tokenizer::TokenizerImpl::get_chat_template() const {
+    std::lock_guard<std::mutex> lock(m_minja_chat_template_cache_mutex);
     return m_chat_template;
 }
 
 std::string Tokenizer::TokenizerImpl::get_original_chat_template() const {
+    std::lock_guard<std::mutex> lock(m_minja_chat_template_cache_mutex);
     return m_original_chat_template;
 }
 

src/cpp/src/tokenizer/tokenizer_impl.hpp

@@ -5,6 +5,8 @@
 #include <filesystem>
 #include <fstream>
 #include <memory>
+#include <mutex>
+#include <unordered_map>
 
 #include "minja/minja.hpp"
 #include "minja/chat-template.hpp"
@@ -43,6 +45,8 @@ class Tokenizer::TokenizerImpl {
     std::string m_eos_token = {};
     std::string m_chat_template = {};
     std::string m_original_chat_template = {};
+    mutable std::mutex m_minja_chat_template_cache_mutex;
+    mutable std::unordered_map<std::string, std::shared_ptr<const minja::chat_template>> m_minja_chat_template_cache;
     std::vector<std::string> m_vocab = {};
     std::shared_ptr<StructuredOutputController> m_structured_output_controller = nullptr;
 
@@ -83,6 +87,9 @@ class Tokenizer::TokenizerImpl {
     std::string get_chat_template() const;
     std::string get_original_chat_template() const;
     std::shared_ptr<StructuredOutputController> get_structured_output_controller(std::optional<int> vocab_size = std::nullopt);
+
+private:
+    std::shared_ptr<const minja::chat_template> get_cached_minja_chat_template(const std::string& chat_template) const;
 };
 
 }  // namespace genai

tests/python_tests/test_llm_pipeline_static.py

@@ -332,6 +332,7 @@ def test_terminate_by_max_number_of_tokens(
 
 @pytest.mark.parametrize("llm_model", MODELS_LIST, indirect=True)
 @pytest.mark.parametrize("npu_config", PIPELINE_CONFIGS, indirect=True)
+@pytest.mark.xfail(reason="Error: KV-Cache is full: num_stored_tokens=319 capacity=319. Ticket 190518")
 def test_terminate_by_out_of_memory(
     llm_model: OVConvertedModelSchema,
     npu_config: dict,