From c84eb397d1e9a6fc27cae2b5f7ef972cc440d431 Mon Sep 17 00:00:00 2001 From: Alexander Suvorov Date: Fri, 10 Jul 2026 15:49:29 +0200 Subject: [PATCH] Split on unicode non whitespace languages --- src/cpp/src/whisper/pipeline_static.cpp | 11 +- src/cpp/src/whisper/whisper.cpp | 15 +- src/cpp/src/whisper/word_level_timestamps.cpp | 134 ++++++++++-------- src/cpp/src/whisper/word_level_timestamps.hpp | 4 +- 4 files changed, 90 insertions(+), 74 deletions(-) diff --git a/src/cpp/src/whisper/pipeline_static.cpp b/src/cpp/src/whisper/pipeline_static.cpp index eadedd8fd9..cc04e16d11 100644 --- a/src/cpp/src/whisper/pipeline_static.cpp +++ b/src/cpp/src/whisper/pipeline_static.cpp @@ -1169,7 +1169,7 @@ WhisperDecodedResults WhisperPipeline::StaticWhisperPipeline::generate( const bool return_timestamps = config.return_timestamps || !is_shortform; WhisperDecodedResults result; - std::vector sot_tokens; + SotTokensResult sot_result; std::vector output_tokens; std::vector segments; @@ -1197,13 +1197,12 @@ WhisperDecodedResults WhisperPipeline::StaticWhisperPipeline::generate( perf_metrics.whisper_raw_metrics); // prepare sot_tokens just once for whole input - if (sot_tokens.empty()) { - auto sot_result = prepare_sot_tokens(hidden_state_tensor, m_models.decoder, config, raw_metrics); - sot_tokens = std::move(sot_result.tokens); + if (sot_result.tokens.empty()) { + sot_result = prepare_sot_tokens(hidden_state_tensor, m_models.decoder, config, raw_metrics); result.language = sot_result.language; } - std::vector chunk_sot_tokens = sot_tokens; + std::vector chunk_sot_tokens = sot_result.tokens; if (!return_timestamps) { chunk_sot_tokens.push_back(config.no_timestamps_token_id); @@ -1262,7 +1261,7 @@ WhisperDecodedResults WhisperPipeline::StaticWhisperPipeline::generate( std::min(m_feature_extractor.nb_max_frames, input_features.n_active_frames - chunk_offset); const auto word_timestamps_processing_start = std::chrono::steady_clock::now(); - const auto word_timestamps = add_word_level_timestamps(sot_tokens, + const auto word_timestamps = add_word_level_timestamps(sot_result, chunk_output_tokens, m_tokenizer, m_models.decoder, diff --git a/src/cpp/src/whisper/whisper.cpp b/src/cpp/src/whisper/whisper.cpp index 5e3670ab3f..8bc48e3448 100644 --- a/src/cpp/src/whisper/whisper.cpp +++ b/src/cpp/src/whisper/whisper.cpp @@ -202,7 +202,7 @@ std::pair decode(std::shared_ptrget_generated_ids()); results.scores.push_back(score); - + ov::genai::GenerationFinishReason finish_reason = sequence->get_finish_reason(); if (sequence_group->handle_stopped() && finish_reason == ov::genai::GenerationFinishReason::NONE) { finish_reason = sequence_group->get_generation_stream()->get_finish_reason(); @@ -315,7 +315,7 @@ WhisperGenerateResult whisper_generate(const ov::genai::WhisperGenerationConfig& // long-form audio processing requires timestamps to be enabled const bool return_timestamps = config.return_timestamps || !is_shortform; - std::vector sot_tokens; + SotTokensResult sot_result; std::vector& output_tokens = result.output_tokens; std::vector segments; @@ -340,15 +340,14 @@ WhisperGenerateResult whisper_generate(const ov::genai::WhisperGenerationConfig& result.perf_metrics.whisper_raw_metrics); // prepare sot_tokens just once for whole input - if (sot_tokens.empty()) { - auto sot_result = prepare_sot_tokens(hidden_state_tensor, decoder, config, raw_metrics); - sot_tokens = std::move(sot_result.tokens); - result.language = std::move(sot_result.language); + if (sot_result.tokens.empty()) { + sot_result = prepare_sot_tokens(hidden_state_tensor, decoder, config, raw_metrics); + result.language = sot_result.language; } std::vector chunk_sot_tokens = ov::genai::get_prompt_tokens(context_tokens, config, chunk_offset); - chunk_sot_tokens.insert(chunk_sot_tokens.end(), sot_tokens.begin(), sot_tokens.end()); + chunk_sot_tokens.insert(chunk_sot_tokens.end(), sot_result.tokens.begin(), sot_result.tokens.end()); if (!return_timestamps) { chunk_sot_tokens.push_back(config.no_timestamps_token_id); @@ -408,7 +407,7 @@ WhisperGenerateResult whisper_generate(const ov::genai::WhisperGenerationConfig& std::min(feature_extractor.nb_max_frames, input_features.n_active_frames - chunk_offset); const auto word_timestamps_processing_start = std::chrono::steady_clock::now(); - const auto word_timestamps = add_word_level_timestamps(sot_tokens, + const auto word_timestamps = add_word_level_timestamps(sot_result, chunk_output_tokens, tokenizer, decoder, diff --git a/src/cpp/src/whisper/word_level_timestamps.cpp b/src/cpp/src/whisper/word_level_timestamps.cpp index a3eb68d309..7d9f217bc4 100644 --- a/src/cpp/src/whisper/word_level_timestamps.cpp +++ b/src/cpp/src/whisper/word_level_timestamps.cpp @@ -320,6 +320,22 @@ std::vector extract_n_frames(const std::vector& alignmen return extracted_tensors; } +std::string trim(const std::string& text) { + std::string result = text; + result.erase(result.begin(), std::find_if(result.begin(), result.end(), [](unsigned char ch) { + return !std::isspace(ch); + })); + + result.erase(std::find_if(result.rbegin(), + result.rend(), + [](unsigned char ch) { + return !std::isspace(ch); + }) + .base(), + result.end()); + return result; +} + std::pair, std::vector>> split_tokens_on_unicode( const std::vector& tokens, ov::genai::Tokenizer& tokenizer) { @@ -354,6 +370,58 @@ std::pair, std::vector>> split_tok return {words, word_tokens}; } +// https://github.com/openai/whisper/blob/v20250625/whisper/tokenizer.py#L311 +std::pair, std::vector>> split_tokens_on_spaces( + const std::vector& tokens, + ov::genai::Tokenizer& tokenizer) { + const auto [subwords, subword_tokens_list] = split_tokens_on_unicode(tokens, tokenizer); + + const int64_t eot = tokenizer.get_eos_token_id(); + + std::vector words; + std::vector> word_tokens; + + for (size_t i = 0; i < subwords.size(); ++i) { + const std::string& subword = subwords[i]; + const std::vector& subword_tokens = subword_tokens_list[i]; + + const bool is_special = subword_tokens.size() && subword_tokens[0] >= eot; + const bool with_space = !subword.empty() && std::isspace(static_cast(subword[0])); + + const std::string trimmed_subword = trim(subword); + const bool is_punctuation = !trimmed_subword.empty() && trimmed_subword.size() == 1 && + std::ispunct(static_cast(trimmed_subword[0])); + + if (words.empty() || is_special || with_space || is_punctuation) { + words.push_back(subword); + word_tokens.push_back(subword_tokens); + } else { + words.back() += subword; + word_tokens.back().insert(word_tokens.back().end(), subword_tokens.begin(), subword_tokens.end()); + } + } + + return {words, word_tokens}; +} + +// Scriptio-continua languages: written without spaces between words. OpenAI groups their +// tokens by unicode codepoint instead of by leading space. +// https://github.com/openai/whisper/blob/v20250625/whisper/tokenizer.py#L294 +bool is_no_space_language(const std::string& language) { + static const std::set no_space_languages = {"zh", "ja", "th", "lo", "my", "yue"}; + + return no_space_languages.count(language) > 0; +} + +// https://github.com/openai/whisper/blob/v20250625/whisper/tokenizer.py#L289 +std::pair, std::vector>> +split_to_word_tokens(const std::vector& tokens, ov::genai::Tokenizer& tokenizer, const std::string& language) { + if (is_no_space_language(language)) { + return split_tokens_on_unicode(tokens, tokenizer); + } + return split_tokens_on_spaces(tokens, tokenizer); +} + std::vector match_words_to_alignment_path( const std::vector& words, const std::vector>& word_tokens, @@ -526,56 +594,6 @@ std::vector merge_punctuations(std::vector, std::vector>> split_tokens_on_spaces( - const std::vector& tokens, - ov::genai::Tokenizer& tokenizer) { - const auto [subwords, subword_tokens_list] = split_tokens_on_unicode(tokens, tokenizer); - - const int64_t eot = tokenizer.get_eos_token_id(); - - std::vector words; - std::vector> word_tokens; - - for (size_t i = 0; i < subwords.size(); ++i) { - const std::string& subword = subwords[i]; - const std::vector& subword_tokens = subword_tokens_list[i]; - - const bool is_special = subword_tokens.size() && subword_tokens[0] >= eot; - const bool with_space = !subword.empty() && std::isspace(static_cast(subword[0])); - - const std::string trimmed_subword = trim(subword); - const bool is_punctuation = !trimmed_subword.empty() && trimmed_subword.size() == 1 && - std::ispunct(static_cast(trimmed_subword[0])); - - if (words.empty() || is_special || with_space || is_punctuation) { - words.push_back(subword); - word_tokens.push_back(subword_tokens); - } else { - words.back() += subword; - word_tokens.back().insert(word_tokens.back().end(), subword_tokens.begin(), subword_tokens.end()); - } - } - - return {words, word_tokens}; -} - std::vector infer_alignments_heads_qks(const std::vector& tokens, std::shared_ptr decoder, const ov::Tensor& hidden_state_tensor, @@ -626,7 +644,7 @@ std::vector infer_alignments_heads_qks(const std::vector& t namespace ov::genai { -std::vector add_word_level_timestamps(const std::vector& sot_tokens, +std::vector add_word_level_timestamps(const SotTokensResult& sot_result, const std::vector& input_tokens, ov::genai::Tokenizer& tokenizer, std::shared_ptr decoder, @@ -644,16 +662,16 @@ std::vector add_word_level_timestamps(const std::v text_tokens.push_back(config.eos_token_id); // [sot_tokens] + [no_timestamps_token] + [text_tokens] + [eos_token] - std::vector infer_tokens = sot_tokens; + std::vector infer_tokens = sot_result.tokens; infer_tokens.push_back(config.no_timestamps_token_id); infer_tokens.insert(infer_tokens.end(), text_tokens.begin(), text_tokens.end()); auto alignment_heads_qks = infer_alignments_heads_qks(infer_tokens, decoder, hidden_state_tensor, config.alignment_heads); - const auto alignment_path = find_alignment_path(alignment_heads_qks, n_active_frames, sot_tokens); + const auto alignment_path = find_alignment_path(alignment_heads_qks, n_active_frames, sot_result.tokens); - const auto [words, word_tokens] = split_tokens_on_spaces(text_tokens, tokenizer); + const auto [words, word_tokens] = split_to_word_tokens(text_tokens, tokenizer, sot_result.language); auto words_timestamps = match_words_to_alignment_path(words, word_tokens, alignment_path, chunk_time_offset); @@ -664,7 +682,7 @@ std::vector add_word_level_timestamps(const std::v return merged_timestamps; } -std::vector add_word_level_timestamps(const std::vector& sot_tokens, +std::vector add_word_level_timestamps(const SotTokensResult& sot_result, const std::vector& input_tokens, ov::genai::Tokenizer& tokenizer, ov::InferRequest& decoder, @@ -682,16 +700,16 @@ std::vector add_word_level_timestamps(const std::v text_tokens.push_back(config.eos_token_id); // [sot_tokens] + [no_timestamps_token] + [text_tokens] + [eos_token] - std::vector infer_tokens = sot_tokens; + std::vector infer_tokens = sot_result.tokens; infer_tokens.push_back(config.no_timestamps_token_id); infer_tokens.insert(infer_tokens.end(), text_tokens.begin(), text_tokens.end()); auto alignment_heads_qks = infer_alignments_heads_qks(infer_tokens, decoder, hidden_state_tensor, config.alignment_heads); - const auto alignment_path = find_alignment_path(alignment_heads_qks, n_active_frames, sot_tokens); + const auto alignment_path = find_alignment_path(alignment_heads_qks, n_active_frames, sot_result.tokens); - const auto [words, word_tokens] = split_tokens_on_spaces(text_tokens, tokenizer); + const auto [words, word_tokens] = split_to_word_tokens(text_tokens, tokenizer, sot_result.language); auto words_timestamps = match_words_to_alignment_path(words, word_tokens, alignment_path, chunk_time_offset); diff --git a/src/cpp/src/whisper/word_level_timestamps.hpp b/src/cpp/src/whisper/word_level_timestamps.hpp index 7ab01f66e0..e1d16a3696 100644 --- a/src/cpp/src/whisper/word_level_timestamps.hpp +++ b/src/cpp/src/whisper/word_level_timestamps.hpp @@ -13,7 +13,7 @@ namespace ov::genai { -std::vector add_word_level_timestamps(const std::vector& sot_tokens, +std::vector add_word_level_timestamps(const SotTokensResult& sot_result, const std::vector& text_tokens, ov::genai::Tokenizer& tokenizer, std::shared_ptr decoder, @@ -22,7 +22,7 @@ std::vector add_word_level_timestamps(const std::v const size_t n_frames, const float chunk_time_offset); -std::vector add_word_level_timestamps(const std::vector& sot_tokens, +std::vector add_word_level_timestamps(const SotTokensResult& sot_result, const std::vector& input_tokens, ov::genai::Tokenizer& tokenizer, ov::InferRequest& decoder,