diff --git a/site/docs/getting-started/introduction.mdx b/site/docs/getting-started/introduction.mdx index f22fceb71c..7040641878 100644 --- a/site/docs/getting-started/introduction.mdx +++ b/site/docs/getting-started/introduction.mdx @@ -15,7 +15,7 @@ This library is friendly to PC and laptop execution, and optimized for resource ## Key Features and Benefits -- **📦 Pre-built Generative AI Pipelines:** Ready-to-use pipelines for text generation (LLMs), image generation (Diffuser-based), speech recognition (Whisper), and visual language models (VLMs). See all [supported use cases](/docs/category/use-cases). +- **📦 Pre-built Generative AI Pipelines:** Ready-to-use pipelines for text generation (LLMs), image generation (Diffuser-based), automatic speech recognition, and visual language models (VLMs). See all [supported use cases](/docs/category/use-cases). - **👣 Minimal Footprint:** Smaller binary size and reduced memory footprint compared to other frameworks. - **🚀 Performance Optimization:** Hardware-specific optimizations for CPU, GPU, and NPU devices. - **👨‍💻 Programming Language Support:** Comprehensive APIs in Python, C++, and Node.js. @@ -37,7 +37,7 @@ Using OpenVINO GenAI typically involves three main steps: Refer to [Model Preparation](/docs/category/model-preparation) for more details. ::: -2. **Pipeline Setup:** Initialize the appropriate pipeline for your task (`LLMPipeline`, `Text2ImagePipeline`, `WhisperPipeline`, `VLMPipeline`, etc.) with the converted model. +2. **Pipeline Setup:** Initialize the appropriate pipeline for your task (`LLMPipeline`, `Text2ImagePipeline`, `ASRPipeline`, `VLMPipeline`, etc.) with the converted model. 3. **Inference:** Run the model with your inputs using the pipeline's simple API. ![OpenVINO GenAI Workflow](/img/openvino-genai-workflow.svg) diff --git a/site/docs/guides/streaming.mdx b/site/docs/guides/streaming.mdx index b6d2df7d7e..662560e603 100644 --- a/site/docs/guides/streaming.mdx +++ b/site/docs/guides/streaming.mdx @@ -7,7 +7,7 @@ sidebar_position: 3 For more interactive UIs during generation, you can stream output tokens. :::info -Streaming is supported for `LLMPipeline`, `VLMPipeline` and `WhisperPipeline`. +Streaming is supported for `LLMPipeline`, `VLMPipeline` and `ASRPipeline`. ::: ## Streaming Function diff --git a/site/docs/supported-models/_components/whisper-models-table/models.ts b/site/docs/supported-models/_components/whisper-models-table/models.ts index ac84eaeb98..574b2d8968 100644 --- a/site/docs/supported-models/_components/whisper-models-table/models.ts +++ b/site/docs/supported-models/_components/whisper-models-table/models.ts @@ -7,6 +7,18 @@ type WhisperModelType = { }; export const WHISPER_MODELS: WhisperModelType[] = [ + { + architecture: 'Qwen3ASRForConditionalGeneration', + models: [ + { + name: 'Qwen3-ASR', + links: [ + 'https://huggingface.co/Qwen/Qwen3-ASR-0.6B', + 'https://huggingface.co/Qwen/Qwen3-ASR-1.7B', + ], + }, + ], + }, { architecture: 'WhisperForConditionalGeneration', models: [ diff --git a/site/docs/supported-models/index.mdx b/site/docs/supported-models/index.mdx index e750f25fc5..bf20d42af9 100644 --- a/site/docs/supported-models/index.mdx +++ b/site/docs/supported-models/index.mdx @@ -107,7 +107,7 @@ The model requires `transformers==5.2` for the export with `optimum-cli`. The model requires `transformers>=4.48` for the export with `optimum-cli`. ::: -## Speech Recognition Models (Whisper-based) +## Speech Recognition Models :::info LoRA Support Speech recognition pipeline does **not** support LoRA adapters. @@ -115,6 +115,14 @@ Speech recognition pipeline does **not** support LoRA adapters. +:::warning Speech Recognition Models Notes +Qwen3-ASR models require the `qwen-asr` package for export with `optimum-cli`: + +```bash +pip install qwen-asr +``` +::: + ## Speech Generation Models :::info LoRA Support diff --git a/site/docs/use-cases/speech-recognition/_sections/_run_model/_code_example_cpp.mdx b/site/docs/use-cases/speech-recognition/_sections/_run_model/_code_example_cpp.mdx index 930cea6856..9cf756a80d 100644 --- a/site/docs/use-cases/speech-recognition/_sections/_run_model/_code_example_cpp.mdx +++ b/site/docs/use-cases/speech-recognition/_sections/_run_model/_code_example_cpp.mdx @@ -1,7 +1,7 @@ import CodeBlock from '@theme/CodeBlock'; -{`#include "openvino/genai/whisper_pipeline.hpp" +{`#include "openvino/genai/automatic_speech_recognition/pipeline.hpp" #include "audio_utils.hpp" #include @@ -9,9 +9,9 @@ int main(int argc, char* argv[]) { std::filesystem::path models_path = argv[1]; std::string wav_file_path = argv[2]; - ov::genai::RawSpeechInput raw_speech = utils::audio::read_wav(wav_file_path); + std::vector raw_speech = utils::audio::read_wav(wav_file_path); - ov::genai::WhisperPipeline pipe(models_path, "${props.device || 'CPU'}"); + ov::genai::ASRPipeline pipe(models_path, "${props.device || 'CPU'}"); auto result = pipe.generate(raw_speech, ov::genai::max_new_tokens(100)); std::cout << result << std::endl; } diff --git a/site/docs/use-cases/speech-recognition/_sections/_run_model/_code_example_python.mdx b/site/docs/use-cases/speech-recognition/_sections/_run_model/_code_example_python.mdx index ccb2dbc834..6ab39032b3 100644 --- a/site/docs/use-cases/speech-recognition/_sections/_run_model/_code_example_python.mdx +++ b/site/docs/use-cases/speech-recognition/_sections/_run_model/_code_example_python.mdx @@ -10,7 +10,7 @@ def read_wav(filepath): raw_speech = read_wav('sample.wav') -pipe = ov_genai.WhisperPipeline(model_path, "${props.device || 'CPU'}") +pipe = ov_genai.ASRPipeline(model_path, "${props.device || 'CPU'}") result = pipe.generate(raw_speech, max_new_tokens=100) print(result) `} diff --git a/site/docs/use-cases/speech-recognition/_sections/_run_model/index.mdx b/site/docs/use-cases/speech-recognition/_sections/_run_model/index.mdx index 2efe8be892..d04272bdcf 100644 --- a/site/docs/use-cases/speech-recognition/_sections/_run_model/index.mdx +++ b/site/docs/use-cases/speech-recognition/_sections/_run_model/index.mdx @@ -4,12 +4,12 @@ import CodeExampleJS from './_code_example_js.mdx'; ## Run Model Using OpenVINO GenAI -OpenVINO GenAI introduces the [`WhisperPipeline`](https://docs.openvino.ai/2026/api/genai_api/_autosummary/openvino_genai.WhisperPipeline.html) pipeline for inference of speech recognition Whisper models. +OpenVINO GenAI introduces the [`ASRPipeline`](https://docs.openvino.ai/2026/api/genai_api/_autosummary/openvino_genai.ASRPipeline.html) pipeline for automatic speech recognition models. You can construct it straight away from the folder with the converted model. It will automatically load the model, tokenizer, detokenizer and default generation configuration. :::info -`WhisperPipeline` expects normalized audio files in WAV format at sampling rate of 16 kHz as input. +`ASRPipeline` expects normalized audio files in WAV format at sampling rate of 16 kHz as input. ::: diff --git a/site/docs/use-cases/speech-recognition/_sections/_usage_options/index.mdx b/site/docs/use-cases/speech-recognition/_sections/_usage_options/index.mdx index 8410d7e946..aaba36829e 100644 --- a/site/docs/use-cases/speech-recognition/_sections/_usage_options/index.mdx +++ b/site/docs/use-cases/speech-recognition/_sections/_usage_options/index.mdx @@ -6,7 +6,7 @@ import Streaming from '@site/docs/use-cases/_shared/_streaming.mdx'; ## Additional Usage Options :::tip -Check out [Python](https://github.com/openvinotoolkit/openvino.genai/tree/master/samples/python/whisper_speech_recognition), [C++](https://github.com/openvinotoolkit/openvino.genai/tree/master/samples/cpp/whisper_speech_recognition), and [JavaScript](https://github.com/openvinotoolkit/openvino.genai/tree/master/samples/js/whisper_speech_recognition) Whisper speech recognition samples. +Check out [Python](https://github.com/openvinotoolkit/openvino.genai/tree/master/samples/python/automatic_speech_recognition), [C++](https://github.com/openvinotoolkit/openvino.genai/tree/master/samples/cpp/automatic_speech_recognition), and [JavaScript](https://github.com/openvinotoolkit/openvino.genai/tree/master/samples/js/whisper_speech_recognition) speech recognition samples. ::: ### Use Different Generation Parameters @@ -19,7 +19,7 @@ Check out [Python](https://github.com/openvinotoolkit/openvino.genai/tree/master ```python import openvino_genai as ov_genai - pipe = ov_genai.WhisperPipeline(model_path, "CPU") + pipe = ov_genai.ASRPipeline(model_path, "CPU") # Get default configuration config = pipe.get_generation_config() @@ -38,7 +38,7 @@ Check out [Python](https://github.com/openvinotoolkit/openvino.genai/tree/master ```cpp int main() { - ov::genai::WhisperPipeline pipe(model_path, "CPU"); + ov::genai::ASRPipeline pipe(model_path, "CPU"); // Get default configuration auto config = pipe.get_generation_config(); @@ -87,7 +87,7 @@ Check out [Python](https://github.com/openvinotoolkit/openvino.genai/tree/master ```python import openvino_genai as ov_genai - pipe = ov_genai.WhisperPipeline(model_path, "CPU") + pipe = ov_genai.ASRPipeline(model_path, "CPU") # Get default generation config config = pipe.get_generation_config() @@ -105,10 +105,10 @@ Check out [Python](https://github.com/openvinotoolkit/openvino.genai/tree/master ```cpp int main() { - ov::genai::WhisperPipeline pipe(model_path, "CPU"); + ov::genai::ASRPipeline pipe(model_path, "CPU"); // Get default generation config - ov::genai::GenerationConfig config = pipe.get_generation_config(); + ov::genai::ASRGenerationConfig config = pipe.get_generation_config(); // Modify parameters config.max_new_tokens = 256; @@ -144,47 +144,51 @@ Check out [Python](https://github.com/openvinotoolkit/openvino.genai/tree/master -### Transcription +:::note +Beam search support depends on the selected model architecture. Qwen3-ASR models do not support beam search decoding. +::: + +#### Transcription -Whisper models can automatically detect the language of the input audio, or you can specify the language to improve accuracy. The detected (or specified) language is always available in the result via the `language` field: +ASR models can automatically detect the language of the input audio, or you can specify the language to improve accuracy. The detected (or specified) language is available in the result via the `languages` field: ```python - pipe = ov_genai.WhisperPipeline(model_path, "CPU") + pipe = ov_genai.ASRPipeline(model_path, "CPU") # Automatic language detection raw_speech = read_wav("speech_sample.wav") result = pipe.generate(raw_speech) - print(result.language) # e.g. "en", "fr", "de" + print(result.languages[0]) # e.g. "en", "fr", "de" - # Explicitly specify language (English) + # Explicitly specify language (English). Use "English" for Qwen3-ASR models. result = pipe.generate(raw_speech, language="<|en|>") - print(result.language) # "en" + print(result.languages[0]) # "en" # French speech sample raw_speech = read_wav("french_sample.wav") result = pipe.generate(raw_speech, language="<|fr|>") - print(result.language) # "fr" + print(result.languages[0]) # "fr" ``` ```cpp int main() { - ov::genai::WhisperPipeline pipe(model_path, "CPU"); + ov::genai::ASRPipeline pipe(model_path, "CPU"); // Automatic language detection auto result = pipe.generate(raw_speech); - std::cout << result.language << "\n"; // e.g. "en", "fr", "de" + std::cout << result.languages[0] << "\n"; // e.g. "en", "fr", "de" - // Explicitly specify language (English) + // Explicitly specify language (English). Use "English" for Qwen3-ASR models. result = pipe.generate(raw_speech, ov::genai::language("<|en|>")); - std::cout << result.language << "\n"; // "en" + std::cout << result.languages[0] << "\n"; // "en" // French speech sample raw_speech = utils::audio::read_wav("french_sample.wav"); result = pipe.generate(raw_speech, ov::genai::language("<|fr|>")); - std::cout << result.language << "\n"; // "fr" + std::cout << result.languages[0] << "\n"; // "fr" } ``` @@ -206,7 +210,13 @@ Whisper models can automatically detect the language of the input audio, or you -### Translation +### Whisper-specific Features + +:::info +These features are specific to Whisper models used with `ASRPipeline`. They may not apply to other ASR model architectures. +::: + +#### Translation By default, Whisper performs transcription, keeping the output in the same language as the input. To translate non-English speech to English, use the `translate` task: @@ -214,7 +224,7 @@ To translate non-English speech to English, use the `translate` task: ```python - pipe = ov_genai.WhisperPipeline(model_path, "CPU") + pipe = ov_genai.ASRPipeline(model_path, "CPU") # Translate French audio to English raw_speech = read_wav("french_sample.wav") @@ -224,7 +234,7 @@ To translate non-English speech to English, use the `translate` task: ```cpp int main() { - ov::genai::WhisperPipeline pipe(model_path, "CPU"); + ov::genai::ASRPipeline pipe(model_path, "CPU"); // Translate French audio to English raw_speech = utils::audio::read_wav("french_sample.wav"); @@ -243,33 +253,33 @@ To translate non-English speech to English, use the `translate` task: -### Timestamps Prediction +#### Timestamps Prediction Whisper can predict timestamps for each segment of speech, which is useful for synchronization or creating subtitles: ```python - pipe = ov_genai.WhisperPipeline(model_path, "CPU") + pipe = ov_genai.ASRPipeline(model_path, "CPU") # Enable timestamp prediction result = pipe.generate(raw_speech, return_timestamps=True) # Print timestamps and text segments - for chunk in result.chunks: + for chunk in result.chunks[0]: print(f"timestamps: [{chunk.start_ts:.2f}, {chunk.end_ts:.2f}] text: {chunk.text}") ``` ```cpp int main() { - ov::genai::WhisperPipeline pipe(model_path, "CPU"); + ov::genai::ASRPipeline pipe(model_path, "CPU"); // Enable timestamp prediction result = pipe.generate(raw_speech, ov::genai::return_timestamps(true)); // Print timestamps and text segments - for (auto& chunk : *result.chunks) { + for (const auto& chunk : (*result.chunks)[0]) { std::cout << "timestamps: [" << chunk.start_ts << ", " << chunk.end_ts << "] text: " << chunk.text << "\n"; } @@ -292,7 +302,7 @@ Whisper can predict timestamps for each segment of speech, which is useful for s -### Word-level Timestamps Prediction +#### Word-level Timestamps Prediction Whisper can predict timestamps for each word of speech, which provides more granular timing information compared to segment-level timestamps. @@ -301,14 +311,14 @@ Whisper can predict timestamps for each word of speech, which provides more gran ```python # Word timestamps require decomposition of cross-attention decoder SDPA layers, # so word_timestamps must be passed to the pipeline constructor (not just in generation config) - pipe = openvino_genai.WhisperPipeline(model_path, "CPU", word_timestamps=True) + pipe = openvino_genai.ASRPipeline(model_path, "CPU", word_timestamps=True) # Enable word-level timestamp prediction result = pipe.generate(raw_speech, word_timestamps=True) # Print word-level timestamps - for word in result.words: - print(f"[{word.start_ts:.2f}, {word.end_ts:.2f}]: {word.word}") + for word in result.words[0]: + print(f"[{word.start_ts:.2f}, {word.end_ts:.2f}]: {word.text}") ``` @@ -316,15 +326,15 @@ Whisper can predict timestamps for each word of speech, which provides more gran int main() { // Word timestamps require decomposition of cross-attention decoder SDPA layers, // so word_timestamps must be passed to the pipeline constructor (not just in generation config) - ov::genai::WhisperPipeline pipeline(model_path, "CPU", ov::genai::word_timestamps(true)); + ov::genai::ASRPipeline pipeline(model_path, "CPU", ov::genai::word_timestamps(true)); // Enable word-level timestamp prediction auto result = pipeline.generate(raw_speech, ov::genai::word_timestamps(true)); // Print word-level timestamps std::cout << std::fixed << std::setprecision(2); - for (auto& word : *result.words) { - std::cout << "[" << word.start_ts << ", " << word.end_ts << "]: " << word.word << "\n"; + for (const auto& word : (*result.words)[0]) { + std::cout << "[" << word.start_ts << ", " << word.end_ts << "]: " << word.text << "\n"; } } ``` @@ -347,15 +357,10 @@ Whisper can predict timestamps for each word of speech, which provides more gran -:::info -NPU device requires `STATIC_PIPELINE=True` property passed to `WhisperPipeline` constructor: -`openvino_genai.WhisperPipeline(model_path, "NPU", word_timestamps=True, STATIC_PIPELINE=True)` -::: - -### Long-Form Audio Processing +#### Long-Form Audio Processing Whisper models are designed for audio segments up to 30 seconds in length. -For longer audio, the OpenVINO GenAI Whisper pipeline automatically handles the processing using a sequential chunking algorithm ("sliding window"): +For longer audio, the OpenVINO GenAI ASR pipeline automatically handles Whisper processing using a sequential chunking algorithm ("sliding window"): 1. The audio is divided into 30-second segments 2. Each segment is processed sequentially @@ -363,7 +368,7 @@ For longer audio, the OpenVINO GenAI Whisper pipeline automatically handles the This happens automatically when you input longer audio files. -### Using Initial Prompts and Hotwords +#### Using Initial Prompts and Hotwords You can improve transcription quality and guide the model's output style by providing initial prompts or hotwords using the following parameters: @@ -377,7 +382,7 @@ Such prompts can be used to steer the model to use particular spellings or style ```python - pipe = ov_genai.WhisperPipeline(model_path, "CPU") + pipe = ov_genai.ASRPipeline(model_path, "CPU") result = pipe.generate(raw_speech) # He has gone and gone for good answered Paul Icrom who... @@ -389,7 +394,7 @@ Such prompts can be used to steer the model to use particular spellings or style ```cpp int main() { - ov::genai::WhisperPipeline pipe(model_path, "CPU"); + ov::genai::ASRPipeline pipe(model_path, "CPU"); auto result = pipeline.generate(raw_speech); // He has gone and gone for good answered Paul Icrom who... @@ -414,7 +419,7 @@ Such prompts can be used to steer the model to use particular spellings or style :::info -For the full list of Whisper generation parameters, refer to the [Whisper Generation Config API](https://docs.openvino.ai/2026/api/genai_api/_autosummary/openvino_genai.WhisperGenerationConfig.html). +For the ASR API, refer to the [ASR Generation Config API](https://docs.openvino.ai/2026/api/genai_api/_autosummary/openvino_genai.ASRGenerationConfig.html). ::: diff --git a/site/docs/use-cases/speech-recognition/index.mdx b/site/docs/use-cases/speech-recognition/index.mdx index 3af226f92c..4808c59bc5 100644 --- a/site/docs/use-cases/speech-recognition/index.mdx +++ b/site/docs/use-cases/speech-recognition/index.mdx @@ -6,21 +6,21 @@ import ConvertModelSection from '../_shared/_convert_model.mdx'; import RunModelSection from './_sections/_run_model/index.mdx'; import UsageOptionsSection from './_sections/_usage_options/index.mdx'; -# Speech Recognition Using Whisper +# Automatic Speech Recognition - Download and convert model (e.g. [openai/whisper-base](https://huggingface.co/openai/whisper-base)) to OpenVINO format from Hugging Face: + Download and convert an automatic speech recognition model (e.g. [openai/whisper-base](https://huggingface.co/openai/whisper-base)) to OpenVINO format from Hugging Face: - + - + - See all supported [Speech Recognition Models](/docs/supported-models/#speech-recognition-models-whisper-based). + See all supported [Speech Recognition Models](/docs/supported-models/#speech-recognition-models). diff --git a/site/src/pages/_sections/HeroSection/PipelinesCarousel/index.tsx b/site/src/pages/_sections/HeroSection/PipelinesCarousel/index.tsx index d2de187670..35b66631b8 100644 --- a/site/src/pages/_sections/HeroSection/PipelinesCarousel/index.tsx +++ b/site/src/pages/_sections/HeroSection/PipelinesCarousel/index.tsx @@ -39,7 +39,7 @@ print(ov_pipe.generate("Describe images", images))`, { title: 'Speech Recognition API', Icon: SoundIcon, - code: `ov_pipe = ov_genai.WhisperPipeline("whisper-base") + code: `ov_pipe = ov_genai.ASRPipeline("whisper-base") print(ov_pipe.generate(read_wav("sample.wav")))`, }, { diff --git a/site/src/pages/_sections/UseCasesSection/components/speech-recognition.tsx b/site/src/pages/_sections/UseCasesSection/components/speech-recognition.tsx index 2b06eef52b..ea8215d68e 100644 --- a/site/src/pages/_sections/UseCasesSection/components/speech-recognition.tsx +++ b/site/src/pages/_sections/UseCasesSection/components/speech-recognition.tsx @@ -9,9 +9,9 @@ import CodeExampleJS from '@site/docs/use-cases/speech-recognition/_sections/_ru export const SpeechRecognition = () => ( - Speech Recognition Using Whisper + Automatic Speech Recognition - Convert speech to text using Whisper models for video transcription, meeting notes, + Convert speech to text using ASR models for video transcription, meeting notes, multilingual audio content processing, and accessibility applications.