Skip to content

[Whisper]: Split on unicode non whitespace languages #4131

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
as-suvorov wants to merge 1 commit into
base: master
Choose a base branch
from
+90 −74
Open

[Whisper]: Split on unicode non whitespace languages #4131

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 5 additions & 6 deletions src/cpp/src/whisper/pipeline_static.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1169,7 +1169,7 @@ WhisperDecodedResults WhisperPipeline::StaticWhisperPipeline::generate(
const bool return_timestamps = config.return_timestamps || !is_shortform;

WhisperDecodedResults result;
std::vector<int64_t> sot_tokens;
SotTokensResult sot_result;
std::vector<int64_t> output_tokens;
std::vector<Segment> segments;

Expand Down Expand Up @@ -1197,13 +1197,12 @@ WhisperDecodedResults WhisperPipeline::StaticWhisperPipeline::generate(
perf_metrics.whisper_raw_metrics);

// prepare sot_tokens just once for whole input
if (sot_tokens.empty()) {
auto sot_result = prepare_sot_tokens(hidden_state_tensor, m_models.decoder, config, raw_metrics);
sot_tokens = std::move(sot_result.tokens);
if (sot_result.tokens.empty()) {
sot_result = prepare_sot_tokens(hidden_state_tensor, m_models.decoder, config, raw_metrics);
result.language = sot_result.language;
}

std::vector<int64_t> chunk_sot_tokens = sot_tokens;
std::vector<int64_t> chunk_sot_tokens = sot_result.tokens;

if (!return_timestamps) {
chunk_sot_tokens.push_back(config.no_timestamps_token_id);
Expand Down Expand Up @@ -1262,7 +1261,7 @@ WhisperDecodedResults WhisperPipeline::StaticWhisperPipeline::generate(
std::min(m_feature_extractor.nb_max_frames, input_features.n_active_frames - chunk_offset);

const auto word_timestamps_processing_start = std::chrono::steady_clock::now();
const auto word_timestamps = add_word_level_timestamps(sot_tokens,
const auto word_timestamps = add_word_level_timestamps(sot_result,
chunk_output_tokens,
m_tokenizer,
m_models.decoder,
Expand Down
15 changes: 7 additions & 8 deletions src/cpp/src/whisper/whisper.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ std::pair<ov::genai::EncodedResults, bool> decode(std::shared_ptr<ov::genai::Whi

results.tokens.push_back(sequence->get_generated_ids());
results.scores.push_back(score);

ov::genai::GenerationFinishReason finish_reason = sequence->get_finish_reason();
if (sequence_group->handle_stopped() && finish_reason == ov::genai::GenerationFinishReason::NONE) {
finish_reason = sequence_group->get_generation_stream()->get_finish_reason();
Expand Down Expand Up @@ -315,7 +315,7 @@ WhisperGenerateResult whisper_generate(const ov::genai::WhisperGenerationConfig&
// long-form audio processing requires timestamps to be enabled
const bool return_timestamps = config.return_timestamps || !is_shortform;

std::vector<int64_t> sot_tokens;
SotTokensResult sot_result;
std::vector<int64_t>& output_tokens = result.output_tokens;
std::vector<Segment> segments;

Expand All @@ -340,15 +340,14 @@ WhisperGenerateResult whisper_generate(const ov::genai::WhisperGenerationConfig&
result.perf_metrics.whisper_raw_metrics);

// prepare sot_tokens just once for whole input
if (sot_tokens.empty()) {
auto sot_result = prepare_sot_tokens(hidden_state_tensor, decoder, config, raw_metrics);
sot_tokens = std::move(sot_result.tokens);
result.language = std::move(sot_result.language);
if (sot_result.tokens.empty()) {
sot_result = prepare_sot_tokens(hidden_state_tensor, decoder, config, raw_metrics);
result.language = sot_result.language;
}

std::vector<int64_t> chunk_sot_tokens = ov::genai::get_prompt_tokens(context_tokens, config, chunk_offset);

chunk_sot_tokens.insert(chunk_sot_tokens.end(), sot_tokens.begin(), sot_tokens.end());
chunk_sot_tokens.insert(chunk_sot_tokens.end(), sot_result.tokens.begin(), sot_result.tokens.end());

if (!return_timestamps) {
chunk_sot_tokens.push_back(config.no_timestamps_token_id);
Expand Down Expand Up @@ -408,7 +407,7 @@ WhisperGenerateResult whisper_generate(const ov::genai::WhisperGenerationConfig&
std::min(feature_extractor.nb_max_frames, input_features.n_active_frames - chunk_offset);

const auto word_timestamps_processing_start = std::chrono::steady_clock::now();
const auto word_timestamps = add_word_level_timestamps(sot_tokens,
const auto word_timestamps = add_word_level_timestamps(sot_result,
chunk_output_tokens,
tokenizer,
decoder,
Expand Down
134 changes: 76 additions & 58 deletions src/cpp/src/whisper/word_level_timestamps.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,22 @@ std::vector<ov::Tensor> extract_n_frames(const std::vector<ov::Tensor>& alignmen
return extracted_tensors;
}

std::string trim(const std::string& text) {
std::string result = text;
result.erase(result.begin(), std::find_if(result.begin(), result.end(), [](unsigned char ch) {
return !std::isspace(ch);
}));

result.erase(std::find_if(result.rbegin(),
result.rend(),
[](unsigned char ch) {
return !std::isspace(ch);
})
.base(),
result.end());
return result;
}

std::pair<std::vector<std::string>, std::vector<std::vector<int64_t>>> split_tokens_on_unicode(
const std::vector<int64_t>& tokens,
ov::genai::Tokenizer& tokenizer) {
Expand Down Expand Up @@ -354,6 +370,58 @@ std::pair<std::vector<std::string>, std::vector<std::vector<int64_t>>> split_tok
return {words, word_tokens};
}

// https://github.com/openai/whisper/blob/v20250625/whisper/tokenizer.py#L311
std::pair<std::vector<std::string>, std::vector<std::vector<int64_t>>> split_tokens_on_spaces(
const std::vector<int64_t>& tokens,
ov::genai::Tokenizer& tokenizer) {
const auto [subwords, subword_tokens_list] = split_tokens_on_unicode(tokens, tokenizer);

const int64_t eot = tokenizer.get_eos_token_id();

std::vector<std::string> words;
std::vector<std::vector<int64_t>> word_tokens;

for (size_t i = 0; i < subwords.size(); ++i) {
const std::string& subword = subwords[i];
const std::vector<int64_t>& subword_tokens = subword_tokens_list[i];

const bool is_special = subword_tokens.size() && subword_tokens[0] >= eot;
const bool with_space = !subword.empty() && std::isspace(static_cast<unsigned char>(subword[0]));

const std::string trimmed_subword = trim(subword);
const bool is_punctuation = !trimmed_subword.empty() && trimmed_subword.size() == 1 &&
std::ispunct(static_cast<unsigned char>(trimmed_subword[0]));

if (words.empty() || is_special || with_space || is_punctuation) {
words.push_back(subword);
word_tokens.push_back(subword_tokens);
} else {
words.back() += subword;
word_tokens.back().insert(word_tokens.back().end(), subword_tokens.begin(), subword_tokens.end());
}
}

return {words, word_tokens};
}

// Scriptio-continua languages: written without spaces between words. OpenAI groups their
// tokens by unicode codepoint instead of by leading space.
// https://github.com/openai/whisper/blob/v20250625/whisper/tokenizer.py#L294
bool is_no_space_language(const std::string& language) {
static const std::set<std::string> no_space_languages = {"zh", "ja", "th", "lo", "my", "yue"};

return no_space_languages.count(language) > 0;
}
Comment on lines +410 to +414

// https://github.com/openai/whisper/blob/v20250625/whisper/tokenizer.py#L289
std::pair<std::vector<std::string>, std::vector<std::vector<int64_t>>>
split_to_word_tokens(const std::vector<int64_t>& tokens, ov::genai::Tokenizer& tokenizer, const std::string& language) {
if (is_no_space_language(language)) {
return split_tokens_on_unicode(tokens, tokenizer);
}
return split_tokens_on_spaces(tokens, tokenizer);
Comment on lines +418 to +422
}

std::vector<ov::genai::WhisperWordTiming> match_words_to_alignment_path(
const std::vector<std::string>& words,
const std::vector<std::vector<int64_t>>& word_tokens,
Expand Down Expand Up @@ -526,56 +594,6 @@ std::vector<ov::genai::WhisperWordTiming> merge_punctuations(std::vector<ov::gen
return filtered_words;
}

std::string trim(const std::string& text) {
std::string result = text;
result.erase(result.begin(), std::find_if(result.begin(), result.end(), [](unsigned char ch) {
return !std::isspace(ch);
}));

result.erase(std::find_if(result.rbegin(),
result.rend(),
[](unsigned char ch) {
return !std::isspace(ch);
})
.base(),
result.end());
return result;
}

// https://github.com/openai/whisper/blob/v20250625/whisper/tokenizer.py#L311
std::pair<std::vector<std::string>, std::vector<std::vector<int64_t>>> split_tokens_on_spaces(
const std::vector<int64_t>& tokens,
ov::genai::Tokenizer& tokenizer) {
const auto [subwords, subword_tokens_list] = split_tokens_on_unicode(tokens, tokenizer);

const int64_t eot = tokenizer.get_eos_token_id();

std::vector<std::string> words;
std::vector<std::vector<int64_t>> word_tokens;

for (size_t i = 0; i < subwords.size(); ++i) {
const std::string& subword = subwords[i];
const std::vector<int64_t>& subword_tokens = subword_tokens_list[i];

const bool is_special = subword_tokens.size() && subword_tokens[0] >= eot;
const bool with_space = !subword.empty() && std::isspace(static_cast<unsigned char>(subword[0]));

const std::string trimmed_subword = trim(subword);
const bool is_punctuation = !trimmed_subword.empty() && trimmed_subword.size() == 1 &&
std::ispunct(static_cast<unsigned char>(trimmed_subword[0]));

if (words.empty() || is_special || with_space || is_punctuation) {
words.push_back(subword);
word_tokens.push_back(subword_tokens);
} else {
words.back() += subword;
word_tokens.back().insert(word_tokens.back().end(), subword_tokens.begin(), subword_tokens.end());
}
}

return {words, word_tokens};
}

std::vector<ov::Tensor> infer_alignments_heads_qks(const std::vector<int64_t>& tokens,
std::shared_ptr<ov::genai::WhisperDecoder> decoder,
const ov::Tensor& hidden_state_tensor,
Expand Down Expand Up @@ -626,7 +644,7 @@ std::vector<ov::Tensor> infer_alignments_heads_qks(const std::vector<int64_t>& t

namespace ov::genai {

std::vector<ov::genai::WhisperWordTiming> add_word_level_timestamps(const std::vector<int64_t>& sot_tokens,
std::vector<ov::genai::WhisperWordTiming> add_word_level_timestamps(const SotTokensResult& sot_result,
const std::vector<int64_t>& input_tokens,
ov::genai::Tokenizer& tokenizer,
std::shared_ptr<ov::genai::WhisperDecoder> decoder,
Expand All @@ -644,16 +662,16 @@ std::vector<ov::genai::WhisperWordTiming> add_word_level_timestamps(const std::v
text_tokens.push_back(config.eos_token_id);

// [sot_tokens] + [no_timestamps_token] + [text_tokens] + [eos_token]
std::vector<int64_t> infer_tokens = sot_tokens;
std::vector<int64_t> infer_tokens = sot_result.tokens;
infer_tokens.push_back(config.no_timestamps_token_id);
infer_tokens.insert(infer_tokens.end(), text_tokens.begin(), text_tokens.end());

auto alignment_heads_qks =
infer_alignments_heads_qks(infer_tokens, decoder, hidden_state_tensor, config.alignment_heads);

const auto alignment_path = find_alignment_path(alignment_heads_qks, n_active_frames, sot_tokens);
const auto alignment_path = find_alignment_path(alignment_heads_qks, n_active_frames, sot_result.tokens);

const auto [words, word_tokens] = split_tokens_on_spaces(text_tokens, tokenizer);
const auto [words, word_tokens] = split_to_word_tokens(text_tokens, tokenizer, sot_result.language);

auto words_timestamps = match_words_to_alignment_path(words, word_tokens, alignment_path, chunk_time_offset);

Expand All @@ -664,7 +682,7 @@ std::vector<ov::genai::WhisperWordTiming> add_word_level_timestamps(const std::v
return merged_timestamps;
}

std::vector<ov::genai::WhisperWordTiming> add_word_level_timestamps(const std::vector<int64_t>& sot_tokens,
std::vector<ov::genai::WhisperWordTiming> add_word_level_timestamps(const SotTokensResult& sot_result,
const std::vector<int64_t>& input_tokens,
ov::genai::Tokenizer& tokenizer,
ov::InferRequest& decoder,
Expand All @@ -682,16 +700,16 @@ std::vector<ov::genai::WhisperWordTiming> add_word_level_timestamps(const std::v
text_tokens.push_back(config.eos_token_id);

// [sot_tokens] + [no_timestamps_token] + [text_tokens] + [eos_token]
std::vector<int64_t> infer_tokens = sot_tokens;
std::vector<int64_t> infer_tokens = sot_result.tokens;
infer_tokens.push_back(config.no_timestamps_token_id);
infer_tokens.insert(infer_tokens.end(), text_tokens.begin(), text_tokens.end());

auto alignment_heads_qks =
infer_alignments_heads_qks(infer_tokens, decoder, hidden_state_tensor, config.alignment_heads);

const auto alignment_path = find_alignment_path(alignment_heads_qks, n_active_frames, sot_tokens);
const auto alignment_path = find_alignment_path(alignment_heads_qks, n_active_frames, sot_result.tokens);

const auto [words, word_tokens] = split_tokens_on_spaces(text_tokens, tokenizer);
const auto [words, word_tokens] = split_to_word_tokens(text_tokens, tokenizer, sot_result.language);

auto words_timestamps = match_words_to_alignment_path(words, word_tokens, alignment_path, chunk_time_offset);

Expand Down
4 changes: 2 additions & 2 deletions src/cpp/src/whisper/word_level_timestamps.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

namespace ov::genai {

std::vector<ov::genai::WhisperWordTiming> add_word_level_timestamps(const std::vector<int64_t>& sot_tokens,
std::vector<ov::genai::WhisperWordTiming> add_word_level_timestamps(const SotTokensResult& sot_result,
const std::vector<int64_t>& text_tokens,
ov::genai::Tokenizer& tokenizer,
std::shared_ptr<ov::genai::WhisperDecoder> decoder,
Expand All @@ -22,7 +22,7 @@ std::vector<ov::genai::WhisperWordTiming> add_word_level_timestamps(const std::v
const size_t n_frames,
const float chunk_time_offset);

std::vector<ov::genai::WhisperWordTiming> add_word_level_timestamps(const std::vector<int64_t>& sot_tokens,
std::vector<ov::genai::WhisperWordTiming> add_word_level_timestamps(const SotTokensResult& sot_result,
const std::vector<int64_t>& input_tokens,
ov::genai::Tokenizer& tokenizer,
ov::InferRequest& decoder,
Expand Down
Loading