diff --git a/samples/js/automatic_speech_recognition/README.md b/samples/js/automatic_speech_recognition/README.md new file mode 100644 index 0000000000..9b32be6238 --- /dev/null +++ b/samples/js/automatic_speech_recognition/README.md @@ -0,0 +1,123 @@ +# Automatic speech recognition sample (JavaScript) + +This example showcases inference of speech recognition models. The application doesn't have many configuration options to encourage the reader to explore and modify the source code. For example, change the device for inference to GPU. The sample features `ASRPipeline` and uses an audio file in wav format as an input source. Audio conversion is performed by a custom helper in `wav_utils.js` (PCM16 mono/stereo at 16 kHz) to align numerical behavior with the C++ and Python sample paths. + +`ASRPipeline` dispatches to the model implementation based on the model's `config.json` (for example Whisper or Qwen3-ASR). + +## Download and convert the model and tokenizers + +The `--upgrade-strategy eager` option is needed to ensure `optimum-intel` is upgraded to the latest version. + +It's not required to install [../../export-requirements.txt](../../export-requirements.txt) for deployment if the model has already been exported. + +```sh +pip install --upgrade-strategy eager -r /samples/requirements.txt +optimum-cli export openvino --trust-remote-code --model openai/whisper-base whisper-base +``` + +## Prepare audio file + +Prepare audio file in wav format with sampling rate 16k Hz. + +You can download example audio file: https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/librispeech_s5/how_are_you_doing_today.wav + +## Run + +From the `samples/js` directory, install dependencies (if not already done): + +```bash +npm install +``` + +If you use the master branch, you may need to [build openvino-genai-node from source](../../src/js/README.md#build-bindings) first. + +Run the sample: + +```bash +node automatic_speech_recognition/automatic_speech_recognition.js whisper-base how_are_you_doing_today.wav +``` + +Optional third argument is the device (default: CPU): + +```bash +node automatic_speech_recognition/automatic_speech_recognition.js whisper-base how_are_you_doing_today.wav GPU +``` + +Output: + +``` + How are you doing today? +timestamps: [0.00, 2.00] text: How are you doing today? +[0.00, 0.xx]: +[0.xx, 0.xx]: How +... +``` + +Refer to the [Supported Models](https://openvinotoolkit.github.io/openvino.genai/docs/supported-models/#speech-recognition-models-whisper-based) for more details. + +# ASR pipeline usage + +```javascript +import { ASRPipeline } from 'openvino-genai-node'; +import { readFileSync } from 'node:fs'; +import { decode } from 'node-wav'; + +const pipeline = await ASRPipeline(modelDir, "CPU"); +const rawSpeechBuffer = readFileSync(audioFilePath); +const rawSpeech = decode(rawSpeechBuffer).channelData[0]; +const result = await pipeline.generate(rawSpeech); +console.log(result.texts[0]); +// How are you doing today? +``` + +### Transcription + +Multilingual models predict the language of the source audio automatically. + +If the source audio language is known in advance, it can be specified in the generation config: + +```javascript +// In the form of "<|en|>" for Whisper models, "English" for Qwen3-ASR models. +const generationConfig = { language: "<|en|>", task: "transcribe" }; +const result = await pipeline.generate(rawSpeech, { generationConfig }); +``` + +### Translation + +By default, Whisper performs the task of speech transcription, where the source audio language is the same as the target text language. To perform speech translation, where the target text is in English, set the task to "translate": + +```javascript +const generationConfig = { task: "translate" }; +const result = await pipeline.generate(rawSpeech, { generationConfig }); +``` + +### Timestamps prediction + +The model can predict timestamps. For sentence-level timestamps, pass the `return_timestamps` argument. The result exposes `chunks` grouped per input, so use `chunks[0]` for a single audio input: + +```javascript +const generationConfig = { return_timestamps: true, language: "<|en|>", task: "transcribe" }; +const result = await pipeline.generate(rawSpeech, { generationConfig }); +for (const chunk of result.chunks?.[0] ?? []) { + console.log(`timestamps: [${chunk.startTs.toFixed(2)}, ${chunk.endTs.toFixed(2)}] text: ${chunk.text}`); +} +``` + +### Word-level timestamps + +Word-level timestamps are supported by Whisper models. Pass `word_timestamps: true` in the pipeline constructor, then in the generation config: + +```javascript +const pipeline = await ASRPipeline(modelDir, "CPU", { word_timestamps: true }); +const generationConfig = { return_timestamps: true, word_timestamps: true, language: "<|en|>", task: "transcribe" }; +const result = await pipeline.generate(rawSpeech, { generationConfig }); +for (const word of result.words?.[0] ?? []) { + console.log(`[${word.startTs.toFixed(2)}, ${word.endTs.toFixed(2)}]: ${word.text}`); +} +``` + +### Initial prompt and hotwords + +The ASR pipeline has `initial_prompt` and `hotwords` generate arguments for Whisper models: +* `initial_prompt`: initial prompt tokens passed as a previous transcription (after `<|startofprev|>` token) to the first processing window +* `hotwords`: hotwords tokens passed as a previous transcription (after `<|startofprev|>` token) to all processing windows diff --git a/samples/js/automatic_speech_recognition/automatic_speech_recognition.js b/samples/js/automatic_speech_recognition/automatic_speech_recognition.js new file mode 100644 index 0000000000..d46df16ea5 --- /dev/null +++ b/samples/js/automatic_speech_recognition/automatic_speech_recognition.js @@ -0,0 +1,90 @@ +// Copyright (C) 2023-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +import { basename } from 'node:path'; +import yargs from 'yargs/yargs'; +import { hideBin } from 'yargs/helpers'; +import { ASRPipeline } from 'openvino-genai-node'; +import { readAudio } from './wav_utils.js'; + +/** + * Parse CLI arguments, run ASR inference and print transcription output. + * @returns {Promise} + */ +async function main() { + const argv = yargs(hideBin(process.argv)) + .scriptName(basename(process.argv[1])) + .command( + '$0 [device]', + 'Run automatic speech recognition on an audio file', + (yargsBuilder) => + yargsBuilder + .positional('model_dir', { + type: 'string', + describe: 'Path to the converted ASR model directory', + demandOption: true, + }) + .positional('audio_file', { + type: 'string', + describe: 'Path to the WAV audio file', + demandOption: true, + }) + .positional('device', { + type: 'string', + describe: 'Device to run the model on (e.g. CPU, GPU)', + default: 'CPU', + }), + ) + .strict() + .help() + .parse(); + + const modelDir = argv.model_dir; + const wavFilePath = argv.audio_file; + const device = argv.device; + + const properties = {}; + if (device === 'NPU' || device.startsWith('GPU')) { + // Cache compiled models on disk for GPU and NPU to save time on the next run. + properties['CACHE_DIR'] = 'asr_cache'; + } + // Word timestamps supported by Whisper models only. + // Must be passed to the ASRPipeline constructor as a property. + properties.word_timestamps = true; + + const pipeline = await ASRPipeline(modelDir, device, properties); + + // If the language is known in advance it can be specified in the generation config. + // In the form of "<|en|>" for Whisper models. Supported by multilingual models only. + // In the form of "English" for Qwen3-ASR models. + // Whisper model parameters (task, return_timestamps, word_timestamps) are ignored for Qwen3-ASR models. + const generationConfig = { + language: '<|en|>', + task: 'transcribe', + return_timestamps: true, + word_timestamps: true, + }; + + // Pipeline expects normalized audio with a sample rate of 16 kHz. + const audioTensor = await readAudio(wavFilePath); + const result = await pipeline.generate(audioTensor, { generationConfig }); + + console.log(result.texts?.[0] ?? ''); + + if (result.chunks?.[0]?.length) { + for (const chunk of result.chunks[0]) { + console.log(`timestamps: [${chunk.startTs.toFixed(2)}, ${chunk.endTs.toFixed(2)}] text: ${chunk.text}`); + } + } + + if (result.words?.[0]?.length) { + for (const word of result.words[0]) { + console.log(`[${word.startTs.toFixed(2)}, ${word.endTs.toFixed(2)}]: ${word.text}`); + } + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/samples/js/automatic_speech_recognition/wav_utils.js b/samples/js/automatic_speech_recognition/wav_utils.js new file mode 100644 index 0000000000..a64b961414 --- /dev/null +++ b/samples/js/automatic_speech_recognition/wav_utils.js @@ -0,0 +1,102 @@ +// Copyright (C) 2023-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +import { readFile } from 'node:fs/promises'; + +function parseWavPcm16Mono(buffer) { + if (buffer.length < 44) { + throw new Error('Invalid WAV payload: file is too small.'); + } + + if (buffer.toString('ascii', 0, 4) !== 'RIFF' || buffer.toString('ascii', 8, 12) !== 'WAVE') { + throw new Error('Invalid WAV payload: RIFF/WAVE header is missing.'); + } + + const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength); + let offset = 12; + + let audioFormat; + let channels; + let sampleRate; + let bitsPerSample; + let dataOffset; + let dataSize; + + while (offset + 8 <= buffer.length) { + const chunkId = buffer.toString('ascii', offset, offset + 4); + const chunkSize = view.getUint32(offset + 4, true); + const chunkDataOffset = offset + 8; + + if (chunkDataOffset + chunkSize > buffer.length) { + throw new Error('Invalid WAV payload: malformed chunk size.'); + } + + if (chunkId === 'fmt ') { + if (chunkSize < 16) { + throw new Error('Invalid WAV payload: fmt chunk is too small.'); + } + audioFormat = view.getUint16(chunkDataOffset, true); + channels = view.getUint16(chunkDataOffset + 2, true); + sampleRate = view.getUint32(chunkDataOffset + 4, true); + bitsPerSample = view.getUint16(chunkDataOffset + 14, true); + } else if (chunkId === 'data') { + dataOffset = chunkDataOffset; + dataSize = chunkSize; + } + + offset = chunkDataOffset + chunkSize + (chunkSize % 2); + } + + if (audioFormat !== 1) { + throw new Error('Unsupported WAV format: only PCM is supported.'); + } + + if (channels !== 1 && channels !== 2) { + throw new Error('WAV file must be mono or stereo.'); + } + + if (sampleRate !== 16000) { + throw new Error(`WAV file must be 16 kHz, but got ${sampleRate}.`); + } + + if (bitsPerSample !== 16) { + throw new Error(`Unsupported WAV bit depth: ${bitsPerSample}. Only 16-bit PCM is supported.`); + } + + if (dataOffset === undefined || dataSize === undefined) { + throw new Error('Invalid WAV payload: missing data chunk.'); + } + + const bytesPerFrame = channels * 2; + const frameCount = Math.floor(dataSize / bytesPerFrame); + const mono = new Float32Array(frameCount); + + for (let index = 0; index < frameCount; index++) { + const frameOffset = dataOffset + index * bytesPerFrame; + if (channels === 1) { + const sample = view.getInt16(frameOffset, true); + mono[index] = sample / 32768.0; + } else { + const left = view.getInt16(frameOffset, true); + const right = view.getInt16(frameOffset + 2, true); + mono[index] = (left + right) / 65536.0; + } + } + + return mono; +} + +/** + * Read WAV file and convert to 16kHz mono Float32Array for the ASR pipeline. + * @param {string} audioPath + * @returns {Promise} + */ +export async function readAudio(audioPath) { + const wavBuffer = await readFile(audioPath); + + if (wavBuffer.length === 0) { + throw new Error('Audio file is empty.'); + } + + return parseWavPcm16Mono(wavBuffer); +} diff --git a/site/docs/bindings/node-js.md b/site/docs/bindings/node-js.md index 864d285e13..db7d43b8dc 100644 --- a/site/docs/bindings/node-js.md +++ b/site/docs/bindings/node-js.md @@ -9,7 +9,7 @@ description: Node.js bindings provide JavaScript/TypeScript API. OpenVINO GenAI provides Node.js bindings that enable you to use generative AI pipelines in JavaScript and TypeScript applications. :::warning API Coverage -Node.js bindings currently provide a subset of the full OpenVINO GenAI API available in C++ and Python. The focus is on core text generation (`LLMPipeline`), vision language models (`VLMPipeline`), text embedding (`TextEmbeddingPipeline`), text reranking (`TextRerankPipeline`), speech recognition (`WhisperPipeline`), speech generation (`Text2SpeechPipeline`), and image generation (`Text2ImagePipeline`, `Image2ImagePipeline`, `InpaintingPipeline`) functionality. +Node.js bindings currently provide a subset of the full OpenVINO GenAI API available in C++ and Python. The focus is on core text generation (`LLMPipeline`), vision language models (`VLMPipeline`), text embedding (`TextEmbeddingPipeline`), text reranking (`TextRerankPipeline`), speech recognition (`WhisperPipeline`, `ASRPipeline`), speech generation (`Text2SpeechPipeline`), and image generation (`Text2ImagePipeline`, `Image2ImagePipeline`, `InpaintingPipeline`) functionality. ::: ## Supported Pipelines and Features @@ -33,6 +33,10 @@ Node.js bindings currently support: - `WhisperPipeline`: Speech recognition from audio input - Streaming support - Timestamps extraction +- `ASRPipeline`: Automatic speech recognition dispatching to Whisper or Qwen3-ASR models + - Streaming support + - Segment- and word-level timestamps extraction + - Detected language reporting - `Text2SpeechPipeline`: Speech generation from text - Optional speaker embedding support - Batch generation diff --git a/src/js/include/addon.hpp b/src/js/include/addon.hpp index 9d1e428fa2..58907ada61 100644 --- a/src/js/include/addon.hpp +++ b/src/js/include/addon.hpp @@ -12,6 +12,7 @@ struct AddonData { Napi::FunctionReference vlm_pipeline; Napi::FunctionReference text_rerank_pipeline; Napi::FunctionReference whisper_pipeline; + Napi::FunctionReference asr_pipeline; Napi::FunctionReference text2image_pipeline; Napi::FunctionReference image2image_pipeline; Napi::FunctionReference inpainting_pipeline; @@ -19,6 +20,7 @@ struct AddonData { Napi::FunctionReference perf_metrics; Napi::FunctionReference vlm_perf_metrics; Napi::FunctionReference whisper_perf_metrics; + Napi::FunctionReference asr_perf_metrics; Napi::FunctionReference text2speech_pipeline; Napi::FunctionReference text2speech_perf_metrics; Napi::FunctionReference text2image_perf_metrics; diff --git a/src/js/include/asr_pipeline/init_worker.hpp b/src/js/include/asr_pipeline/init_worker.hpp new file mode 100644 index 0000000000..92e9142a50 --- /dev/null +++ b/src/js/include/asr_pipeline/init_worker.hpp @@ -0,0 +1,31 @@ +// Copyright (C) 2023-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include + +#include "openvino/genai/automatic_speech_recognition/pipeline.hpp" + +class ASRInitWorker : public Napi::AsyncWorker { +public: + ASRInitWorker(Napi::Function& callback, + std::shared_ptr& pipe, + std::shared_ptr> is_initializing, + std::filesystem::path model_path, + std::string device, + ov::AnyMap properties); + virtual ~ASRInitWorker() {} + void Execute() override; + void OnOK() override; + void OnError(const Napi::Error& e) override; + +private: + std::shared_ptr& pipe; + std::shared_ptr> is_initializing; + std::filesystem::path model_path; + std::string device; + ov::AnyMap properties; +}; diff --git a/src/js/include/asr_pipeline/perf_metrics.hpp b/src/js/include/asr_pipeline/perf_metrics.hpp new file mode 100644 index 0000000000..2933c88f97 --- /dev/null +++ b/src/js/include/asr_pipeline/perf_metrics.hpp @@ -0,0 +1,24 @@ +// Copyright (C) 2023-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include "include/base/perf_metrics.hpp" +#include "openvino/genai/automatic_speech_recognition/perf_metrics.hpp" + +class ASRPerfMetricsWrapper : public BasePerfMetricsWrapper { +public: + ASRPerfMetricsWrapper(const Napi::CallbackInfo& info); + + static Napi::Function get_class(Napi::Env env); + static Napi::Object wrap(Napi::Env env, const ov::genai::ASRPerfMetrics& metrics); + + Napi::Value get_features_extraction_duration(const Napi::CallbackInfo& info); + Napi::Value get_word_level_timestamps_processing_duration(const Napi::CallbackInfo& info); + Napi::Value get_encode_inference_duration(const Napi::CallbackInfo& info); + Napi::Value get_decode_inference_duration(const Napi::CallbackInfo& info); + Napi::Value get_raw_metrics(const Napi::CallbackInfo& info); + Napi::Value get_asr_raw_metrics(const Napi::CallbackInfo& info); +}; diff --git a/src/js/include/asr_pipeline/pipeline_wrapper.hpp b/src/js/include/asr_pipeline/pipeline_wrapper.hpp new file mode 100644 index 0000000000..73d103bd23 --- /dev/null +++ b/src/js/include/asr_pipeline/pipeline_wrapper.hpp @@ -0,0 +1,25 @@ +// Copyright (C) 2023-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +#include "openvino/genai/automatic_speech_recognition/pipeline.hpp" + +class ASRPipelineWrapper : public Napi::ObjectWrap { +public: + ASRPipelineWrapper(const Napi::CallbackInfo& info); + static Napi::Function get_class(Napi::Env env); + Napi::Value init(const Napi::CallbackInfo& info); + Napi::Value generate(const Napi::CallbackInfo& info); + Napi::Value get_tokenizer(const Napi::CallbackInfo& info); + Napi::Value get_generation_config(const Napi::CallbackInfo& info); + Napi::Value set_generation_config(const Napi::CallbackInfo& info); + +private: + std::shared_ptr pipe = nullptr; + std::shared_ptr> is_initializing = std::make_shared>(false); + std::shared_ptr> is_generating = std::make_shared>(false); +}; diff --git a/src/js/include/helper.hpp b/src/js/include/helper.hpp index c0667d7734..a1e4b9a104 100644 --- a/src/js/include/helper.hpp +++ b/src/js/include/helper.hpp @@ -10,6 +10,7 @@ #include "openvino/core/type/element_type.hpp" #include "openvino/genai/llm_pipeline.hpp" +#include "openvino/genai/automatic_speech_recognition/pipeline.hpp" #include "openvino/genai/image_generation/generation_config.hpp" #include "openvino/genai/image_generation/image_generation_perf_metrics.hpp" #include "openvino/genai/rag/text_embedding_pipeline.hpp" @@ -162,6 +163,9 @@ template <> ov::genai::WhisperPerfMetrics& unwrap(const Napi::Env& env, const Napi::Value& value); +template <> +ov::genai::ASRPerfMetrics& unwrap(const Napi::Env& env, const Napi::Value& value); + template <> ov::genai::SpeechGenerationPerfMetrics& unwrap(const Napi::Env& env, const Napi::Value& value); @@ -281,6 +285,11 @@ Napi::Value cpp_to_js( const Napi::Env& env, const ov::genai::WhisperGenerationConfig& config); +/** @brief A template specialization for TargetType Napi::Value and SourceType ov::genai::ASRGenerationConfig */ +template <> +Napi::Value cpp_to_js(const Napi::Env& env, + const ov::genai::ASRGenerationConfig& config); + /** @brief A template specialization for TargetType Napi::Value and SourceType ov::genai::SpeechGenerationConfig */ template <> Napi::Value cpp_to_js( @@ -343,4 +352,6 @@ Napi::Object to_vlm_decoded_result(const Napi::Env& env, const ov::genai::VLMDec Napi::Object to_whisper_decoded_result(const Napi::Env& env, const ov::genai::WhisperDecodedResults& results); +Napi::Object to_asr_decoded_result(const Napi::Env& env, const ov::genai::ASRDecodedResults& results); + Napi::Object to_text2speech_decoded_result(const Napi::Env& env, const ov::genai::Text2SpeechDecodedResults& results); diff --git a/src/js/lib/addon.ts b/src/js/lib/addon.ts index ab169d5e05..d134d97b81 100644 --- a/src/js/lib/addon.ts +++ b/src/js/lib/addon.ts @@ -23,6 +23,8 @@ import { LLMPipelineProperties, WhisperGenerationConfig, WhisperPipelineProperties, + ASRGenerationConfig, + ASRPipelineProperties, SpeechGenerationConfig, ImageGenerationConfig, ImageGenerationCallback, @@ -35,10 +37,12 @@ import { VLMPerfMetrics, PerfMetrics, WhisperPerfMetrics, + ASRPerfMetrics, ImageGenerationPerfMetrics, Text2SpeechPerfMetrics, } from "./perfMetrics.js"; import type { WhisperDecodedResultChunk, WhisperWordTiming } from "./decodedResults.js"; +import type { ASRDecodedResultChunk } from "./decodedResults.js"; export type EmbeddingResult = Float32Array | Int8Array | Uint8Array; export type EmbeddingResults = Float32Array[] | Int8Array[] | Uint8Array[]; @@ -186,6 +190,35 @@ export interface WhisperPipeline { setGenerationConfig(config: WhisperGenerationConfig): void; } +export interface ASRPipeline { + new (): ASRPipeline; + init( + modelPath: string, + device: string, + properties: ASRPipelineProperties, + callback: (err: Error | null) => void, + ): void; + generate( + rawSpeech: Float32Array | number[], + generationConfig: ASRGenerationConfig, + streamer: ((chunk: string) => StreamingStatus) | undefined, + callback: ( + err: Error | null, + result: { + texts: string[]; + scores: number[]; + languages: string[]; + perfMetrics: ASRPerfMetrics; + chunks?: ASRDecodedResultChunk[][]; + words?: ASRDecodedResultChunk[][]; + }, + ) => void, + ): void; + getTokenizer(): ITokenizer; + getGenerationConfig(): Partial; + setGenerationConfig(config: ASRGenerationConfig): void; +} + export interface VLMPipeline { new (): VLMPipeline; init( @@ -312,6 +345,7 @@ interface OpenVINOGenAIAddon { LLMPipeline: LLMPipeline; VLMPipeline: VLMPipeline; WhisperPipeline: WhisperPipeline; + ASRPipeline: ASRPipeline; Text2ImagePipeline: Text2ImagePipeline; Image2ImagePipeline: Image2ImagePipeline; InpaintingPipeline: InpaintingPipeline; @@ -350,6 +384,7 @@ export const { LLMPipeline, VLMPipeline, WhisperPipeline, + ASRPipeline, Text2ImagePipeline, Image2ImagePipeline, InpaintingPipeline, diff --git a/src/js/lib/decodedResults.ts b/src/js/lib/decodedResults.ts index 1b6d2cde58..8a564e3a07 100644 --- a/src/js/lib/decodedResults.ts +++ b/src/js/lib/decodedResults.ts @@ -7,6 +7,7 @@ import { PerfMetrics, VLMPerfMetrics, WhisperPerfMetrics, + ASRPerfMetrics, Text2SpeechPerfMetrics, } from "./perfMetrics.js"; import { GenerationFinishReason } from "./utils.js"; @@ -118,6 +119,44 @@ export class WhisperDecodedResults extends DecodedResults { override perfMetrics: WhisperPerfMetrics; } +/** + * Time-aligned text chunk for ASR results (segment- or word-level). + * Returned when `return_timestamps` (segments) or `word_timestamps` (words) is enabled. + */ +export type ASRDecodedResultChunk = { + /** Chunk text. */ + text: string; + /** Chunk start time in seconds. */ + startTs: number; + /** Chunk end time in seconds (-1 if not predicted by the model). */ + endTs: number; + /** Token identifiers composing the chunk as `BigInt64Array`. */ + tokenIds?: BigInt64Array; +}; + +/** + * Result of ASRPipeline.generate() with texts, scores, detected languages, + * perf metrics, and optional segment- and word-level timestamps. + * + * `chunks` and `words` are nested per input: `chunks[inputIndex][chunkIndex]`. + */ +export class ASRDecodedResults extends DecodedResults { + constructor( + texts: string[], + scores: number[], + perfMetrics: ASRPerfMetrics, + public languages: string[] = [], + public chunks?: ASRDecodedResultChunk[][], + public words?: ASRDecodedResultChunk[][], + ) { + super(texts, scores, perfMetrics, []); + this.perfMetrics = perfMetrics; + } + + /** ASR-specific performance metrics. */ + override perfMetrics: ASRPerfMetrics; +} + /** * Result of Text2SpeechPipeline.generate() with audio tensors and perf metrics. * Each element in `speeches` is an audio waveform tensor sampled at 16 kHz. diff --git a/src/js/lib/index.ts b/src/js/lib/index.ts index 6e5adfe697..e090d1fe3b 100644 --- a/src/js/lib/index.ts +++ b/src/js/lib/index.ts @@ -9,6 +9,7 @@ import { TextRerankPipelineOptions, } from "./pipelines/textRerankPipeline.js"; import { WhisperPipeline as Whisper } from "./pipelines/whisperPipeline.js"; +import { ASRPipeline as ASR } from "./pipelines/asrPipeline.js"; import { Text2ImagePipeline as Text2Image } from "./pipelines/text2ImagePipeline.js"; import { Image2ImagePipeline as Image2Image } from "./pipelines/image2ImagePipeline.js"; import { InpaintingPipeline as Inpainting } from "./pipelines/inpaintingPipeline.js"; @@ -17,6 +18,7 @@ import { LLMPipelineProperties, VLMPipelineProperties, WhisperPipelineProperties, + ASRPipelineProperties, Text2ImagePipelineProperties, Image2ImagePipelineProperties, InpaintingPipelineProperties, @@ -83,6 +85,17 @@ class PipelineFactory { return pipeline; } + static async ASRPipeline( + modelPath: string, + device: string = "CPU", + properties: ASRPipelineProperties = {}, + ) { + const pipeline = new ASR(modelPath, device, properties); + await pipeline.init(); + + return pipeline; + } + static async Text2ImagePipeline( modelPath: string, device: string = "CPU", @@ -134,6 +147,7 @@ export const { TextEmbeddingPipeline, TextRerankPipeline, WhisperPipeline, + ASRPipeline, Text2ImagePipeline, Image2ImagePipeline, InpaintingPipeline, @@ -143,13 +157,16 @@ export { DecodedResults, VLMDecodedResults, WhisperDecodedResults, + ASRDecodedResults, Text2SpeechDecodedResults, } from "./decodedResults.js"; export type { WhisperDecodedResultChunk, WhisperWordTiming } from "./decodedResults.js"; +export type { ASRDecodedResultChunk } from "./decodedResults.js"; export { PerfMetrics, VLMPerfMetrics, WhisperPerfMetrics, + ASRPerfMetrics, ImageGenerationPerfMetrics, Text2ImagePerfMetrics, Text2SpeechPerfMetrics, diff --git a/src/js/lib/perfMetrics.ts b/src/js/lib/perfMetrics.ts index e7b3b593b2..d6f3b09c24 100644 --- a/src/js/lib/perfMetrics.ts +++ b/src/js/lib/perfMetrics.ts @@ -51,6 +51,18 @@ export type WhisperRawMetrics = { wordLevelTimestampsProcessingDurations: number[]; }; +/** Structure with raw performance metrics for ASR generation. */ +export type ASRRawMetrics = { + /** Durations for features extraction in milliseconds. */ + featuresExtractionDurations: number[]; + /** Durations for word-level timestamps processing in milliseconds. */ + wordLevelTimestampsProcessingDurations: number[]; + /** Durations for encoder inference in milliseconds. */ + encodeInferenceDurations: number[]; + /** Durations for decoder inference in milliseconds. */ + decodeInferenceDurations: number[]; +}; + /** * Holds performance metrics for each generate call. * @@ -146,6 +158,33 @@ export interface WhisperPerfMetrics extends PerfMetrics { add(other: WhisperPerfMetrics): this; } +/** + * Holds performance metrics for each ASR generate call. + * + * ASRPerfMetrics extends PerfMetrics with speech-recognition-specific metrics: + * - Features extraction duration, ms + * - Word-level timestamps processing duration, ms + * - Encoder inference duration, ms + * - Decoder inference duration, ms + */ +export interface ASRPerfMetrics extends PerfMetrics { + /** Returns the mean and standard deviation of features extraction duration in milliseconds. */ + getFeaturesExtractionDuration(): MeanStdPair; + /** Returns the mean and standard deviation of word-level timestamps processing duration in milliseconds. */ + getWordLevelTimestampsProcessingDuration(): MeanStdPair; + /** Returns the mean and standard deviation of encoder inference duration in milliseconds. */ + getEncodeInferenceDuration(): MeanStdPair; + /** Returns the mean and standard deviation of decoder inference duration in milliseconds. */ + getDecodeInferenceDuration(): MeanStdPair; + /** ASR-specific raw metrics */ + asrRawMetrics: ASRRawMetrics; + + /** Adds the metrics from another ASRPerfMetrics object to this one. + * @returns The current ASRPerfMetrics instance. + */ + add(other: ASRPerfMetrics): this; +} + /** * Holds performance metrics for each Text2Speech generate call. * diff --git a/src/js/lib/pipelines/asrPipeline.ts b/src/js/lib/pipelines/asrPipeline.ts new file mode 100644 index 0000000000..8a3b6e9cf3 --- /dev/null +++ b/src/js/lib/pipelines/asrPipeline.ts @@ -0,0 +1,207 @@ +// Copyright (C) 2023-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +import util from "node:util"; +import { ASRPipeline as ASRPipelineWrapper } from "../addon.js"; +import { ASRDecodedResultChunk, ASRDecodedResults } from "../decodedResults.js"; +import { ASRGenerationConfig, ASRPipelineProperties, StreamingStatus } from "../utils.js"; +import { Tokenizer } from "../tokenizer.js"; +import { ASRPerfMetrics } from "../perfMetrics.js"; + +export type RawSpeechInput = Float32Array | number[]; + +/** + * Options for ASR generation methods. + */ +export type ASRGenerateOptions = { + /** Generation configuration (e.g. language, task, return_timestamps). */ + generationConfig?: ASRGenerationConfig; + /** Callback invoked with each decoded text chunk; return StreamingStatus to control generation. */ + streamer?: (chunk: string) => StreamingStatus; +}; + +/** + * Pipeline for automatic speech recognition. + * + * Dispatches to the model implementation based on the model's `config.json` + * (e.g. Whisper or Qwen3-ASR). Expects raw audio normalized to approximately + * [-1, 1] at 16 kHz sample rate. Use a WAV file or decode audio to Float32Array + * before calling generate(). + */ +export class ASRPipeline { + protected readonly modelPath: string; + protected readonly device: string; + protected pipeline: ASRPipelineWrapper | null = null; + protected readonly properties: ASRPipelineProperties; + + /** + * Construct an ASR pipeline from a folder containing model IRs and tokenizer. + * @param modelPath - Path to the folder with model IRs and tokenizer. + * @param device - Inference device (e.g. "CPU", "GPU"). + * @param properties - Device and pipeline properties (e.g. word_timestamps: true, CACHE_DIR: "cache"). + */ + constructor(modelPath: string, device: string, properties: ASRPipelineProperties = {}) { + this.modelPath = modelPath; + this.device = device; + this.properties = properties; + } + + /** + * Load the pipeline. Must be called once before generate(). + */ + async init(): Promise { + const pipeline = new ASRPipelineWrapper(); + const initPromise = util.promisify(pipeline.init.bind(pipeline)); + await initPromise(this.modelPath, this.device, this.properties); + this.pipeline = pipeline; + } + + /** + * Stream speech recognition results as an async iterator. + * The iterator yields decoded text chunks during generation. + * When generation finishes, the full decoded text is returned as the final + * iterator value (`done: true`). This value is not available through + * `for await...of`; call `next()` directly to read it. + * + * For custom streaming control, use {@link generate} with a streamer callback instead. + * + * @param rawSpeech - Audio samples as Float32Array or number[], normalized to ~[-1, 1], 16 kHz. + * @param options - Optional generation config (e.g. language, task, return_timestamps). + * @returns Async iterator that yields decoded text chunks as strings. + */ + stream(rawSpeech: RawSpeechInput, options?: ASRGenerateOptions): AsyncIterableIterator { + if (!this.pipeline) throw new Error("ASRPipeline is not initialized"); + const generationConfig = options?.generationConfig ?? {}; + + let streamingStatus = StreamingStatus.RUNNING; + let handledError: Error | null; + const queue: { done: boolean; chunk: string }[] = []; + type ResolveFunction = (arg: { value: string; done: boolean }) => void; + let resolvePromise: ResolveFunction | null = null; + let rejectPromise: ((reason?: unknown) => void) | null = null; + + const callback = ( + error: Error | null, + result: { + texts: string[]; + scores: number[]; + languages: string[]; + perfMetrics: ASRPerfMetrics; + chunks?: ASRDecodedResultChunk[][]; + words?: ASRDecodedResultChunk[][]; + }, + ) => { + if (error) { + if (rejectPromise) { + rejectPromise(error); + resolvePromise = null; + rejectPromise = null; + } else { + handledError = error; + } + } else { + const fullText = result.texts?.[0] ?? ""; + if (resolvePromise) { + resolvePromise({ done: true, value: fullText }); + resolvePromise = null; + rejectPromise = null; + } else { + queue.push({ done: true, chunk: fullText }); + } + } + }; + + const streamer = (chunk: string): StreamingStatus => { + if (resolvePromise) { + resolvePromise({ done: false, value: chunk }); + resolvePromise = null; + rejectPromise = null; + } else { + queue.push({ done: false, chunk }); + } + return streamingStatus; + }; + + this.pipeline.generate(rawSpeech, generationConfig, streamer, callback); + + return { + async next() { + if (handledError) { + const error = handledError; + handledError = null; + return Promise.reject(error); + } + const data = queue.shift(); + if (data) { + return { value: data.chunk, done: data.done }; + } + return new Promise>((resolve, reject) => { + resolvePromise = resolve; + rejectPromise = reject; + }); + }, + async return() { + streamingStatus = StreamingStatus.CANCEL; + return { done: true, value: "" }; + }, + [Symbol.asyncIterator]() { + return this; + }, + }; + } + + /** + * Run speech recognition with optional streaming. + * + * For simple streaming use cases, consider using {@link stream}, which provides + * a convenient async iterator interface. + * + * @param rawSpeech - Audio samples as Float32Array or number[], normalized to ~[-1, 1], 16 kHz. + * @param options - Optional parameters. + * @param options.generationConfig - Generation config (e.g., language, task, return_timestamps). + * @param options.streamer - Optional callback invoked for each decoded chunk. + * - Return a `StreamingStatus` flag to indicate whether generation should be stopped or cancelled + * @returns Decoded texts, scores, detected languages, optional timestamps, and perf metrics. + */ + async generate( + rawSpeech: RawSpeechInput, + options: ASRGenerateOptions = {}, + ): Promise { + if (!this.pipeline) throw new Error("ASRPipeline is not initialized"); + const { generationConfig, streamer } = options; + const generatePromise = util.promisify(this.pipeline.generate.bind(this.pipeline)); + const res = await generatePromise(rawSpeech, generationConfig ?? {}, streamer); + return new ASRDecodedResults( + res.texts, + res.scores, + res.perfMetrics, + res.languages, + res.chunks, + res.words, + ); + } + + /** + * Get the pipeline tokenizer. + */ + getTokenizer(): Tokenizer { + if (!this.pipeline) throw new Error("ASRPipeline is not initialized"); + return this.pipeline.getTokenizer(); + } + + /** + * Get current generation config (language, task, return_timestamps, etc.). + */ + getGenerationConfig(): Partial { + if (!this.pipeline) throw new Error("ASRPipeline is not initialized"); + return this.pipeline.getGenerationConfig(); + } + + /** + * Update generation config (e.g. language, task, return_timestamps). + */ + setGenerationConfig(config: ASRGenerationConfig): void { + if (!this.pipeline) throw new Error("ASRPipeline is not initialized"); + this.pipeline.setGenerationConfig(config); + } +} diff --git a/src/js/lib/utils.ts b/src/js/lib/utils.ts index a2419bef49..2d3765234a 100644 --- a/src/js/lib/utils.ts +++ b/src/js/lib/utils.ts @@ -461,6 +461,16 @@ export type WhisperGenerationConfig = GenerationConfig & { suppress_tokens?: Uint[]; }; +/** + * Generation config for ASRPipeline. Extends the Whisper generation config with + * Qwen3-ASR-specific options. Whisper-specific fields are ignored by Qwen3-ASR models + * and vice versa. + */ +export type ASRGenerationConfig = WhisperGenerationConfig & { + /** Context passed as a previous transcription to steer spelling or style. Supported by Qwen3-ASR models only. */ + context?: string; +}; + /** Generation config for Text2SpeechPipeline. Extends GenerationConfig with speech-specific options. */ export type SpeechGenerationConfig = GenerationConfig & { /** Minimum ratio of output length to input text length; prevents output that's too short. */ @@ -508,6 +518,10 @@ export type WhisperPipelineProperties = { schedulerConfig?: SchedulerConfig; } & Record; +export type ASRPipelineProperties = { + schedulerConfig?: SchedulerConfig; +} & Record; + export type ImageGenerationConfig = { /** Additional prompt for the second text encoder. */ prompt_2?: string; diff --git a/src/js/src/addon.cpp b/src/js/src/addon.cpp index 854cb7e5bf..c333739f34 100644 --- a/src/js/src/addon.cpp +++ b/src/js/src/addon.cpp @@ -7,6 +7,8 @@ #include +#include "include/asr_pipeline/perf_metrics.hpp" +#include "include/asr_pipeline/pipeline_wrapper.hpp" #include "include/chat_history.hpp" #include "include/image2image_pipeline/pipeline_wrapper.hpp" #include "include/inpainting_pipeline/pipeline_wrapper.hpp" @@ -86,6 +88,7 @@ Napi::Object init_module(Napi::Env env, Napi::Object exports) { &TextRerankPipelineWrapper::get_class, addon_data->text_rerank_pipeline); init_class(env, exports, "WhisperPipeline", &WhisperPipelineWrapper::get_class, addon_data->whisper_pipeline); + init_class(env, exports, "ASRPipeline", &ASRPipelineWrapper::get_class, addon_data->asr_pipeline); init_class(env, exports, "Tokenizer", &TokenizerWrapper::get_class, addon_data->tokenizer); init_class(env, exports, "PerfMetrics", &PerfMetricsWrapper::get_class, addon_data->perf_metrics); init_class(env, exports, "VLMPerfMetrics", &VLMPerfMetricsWrapper::get_class, addon_data->vlm_perf_metrics); @@ -94,6 +97,7 @@ Napi::Object init_module(Napi::Env env, Napi::Object exports) { "WhisperPerfMetrics", &WhisperPerfMetricsWrapper::get_class, addon_data->whisper_perf_metrics); + init_class(env, exports, "ASRPerfMetrics", &ASRPerfMetricsWrapper::get_class, addon_data->asr_perf_metrics); init_class(env, exports, "Text2ImagePerfMetrics", diff --git a/src/js/src/asr_pipeline/init_worker.cpp b/src/js/src/asr_pipeline/init_worker.cpp new file mode 100644 index 0000000000..c578a94f05 --- /dev/null +++ b/src/js/src/asr_pipeline/init_worker.cpp @@ -0,0 +1,33 @@ +// Copyright (C) 2023-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#include "include/asr_pipeline/init_worker.hpp" + +#include "include/helper.hpp" + +ASRInitWorker::ASRInitWorker(Napi::Function& callback, + std::shared_ptr& pipe, + std::shared_ptr> is_initializing, + std::filesystem::path model_path, + std::string device, + ov::AnyMap properties) + : Napi::AsyncWorker(callback), + pipe(pipe), + is_initializing(is_initializing), + model_path(std::move(model_path)), + device(std::move(device)), + properties(std::move(properties)) {} + +void ASRInitWorker::Execute() { + this->pipe = std::make_shared(this->model_path, this->device, this->properties); +} + +void ASRInitWorker::OnOK() { + *this->is_initializing = false; + Callback().Call({Env().Null()}); +} + +void ASRInitWorker::OnError(const Napi::Error& e) { + *this->is_initializing = false; + Callback().Call({Napi::Error::New(Env(), e.Message()).Value()}); +} diff --git a/src/js/src/asr_pipeline/perf_metrics.cpp b/src/js/src/asr_pipeline/perf_metrics.cpp new file mode 100644 index 0000000000..64adc72e05 --- /dev/null +++ b/src/js/src/asr_pipeline/perf_metrics.cpp @@ -0,0 +1,81 @@ +// Copyright (C) 2023-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#include "include/asr_pipeline/perf_metrics.hpp" + +#include "include/addon.hpp" +#include "include/helper.hpp" + +using ov::genai::common_bindings::utils::get_ms; + +ASRPerfMetricsWrapper::ASRPerfMetricsWrapper(const Napi::CallbackInfo& info) + : BasePerfMetricsWrapper(info) {} + +Napi::Function ASRPerfMetricsWrapper::get_class(Napi::Env env) { + auto properties = BasePerfMetricsWrapper::get_class_properties(); + properties.push_back( + InstanceMethod("getFeaturesExtractionDuration", &ASRPerfMetricsWrapper::get_features_extraction_duration)); + properties.push_back(InstanceMethod("getWordLevelTimestampsProcessingDuration", + &ASRPerfMetricsWrapper::get_word_level_timestamps_processing_duration)); + properties.push_back( + InstanceMethod("getEncodeInferenceDuration", &ASRPerfMetricsWrapper::get_encode_inference_duration)); + properties.push_back( + InstanceMethod("getDecodeInferenceDuration", &ASRPerfMetricsWrapper::get_decode_inference_duration)); + properties.push_back(InstanceAccessor("asrRawMetrics", &ASRPerfMetricsWrapper::get_asr_raw_metrics, nullptr)); + return DefineClass(env, "ASRPerfMetrics", properties); +} + +Napi::Object ASRPerfMetricsWrapper::wrap(Napi::Env env, const ov::genai::ASRPerfMetrics& metrics) { + const auto& prototype = env.GetInstanceData()->asr_perf_metrics; + OPENVINO_ASSERT(prototype, "Invalid pointer to prototype."); + auto obj = prototype.New({}); + const auto m_ptr = Napi::ObjectWrap::Unwrap(obj); + m_ptr->_metrics = metrics; + return obj; +} + +Napi::Value ASRPerfMetricsWrapper::get_features_extraction_duration(const Napi::CallbackInfo& info) { + VALIDATE_ARGS_COUNT(info, 0, "getFeaturesExtractionDuration()"); + return perf_utils::create_mean_std_pair(info.Env(), _metrics.get_features_extraction_duration()); +} + +Napi::Value ASRPerfMetricsWrapper::get_word_level_timestamps_processing_duration(const Napi::CallbackInfo& info) { + VALIDATE_ARGS_COUNT(info, 0, "getWordLevelTimestampsProcessingDuration()"); + return perf_utils::create_mean_std_pair(info.Env(), _metrics.get_word_level_timestamps_processing_duration()); +} + +Napi::Value ASRPerfMetricsWrapper::get_encode_inference_duration(const Napi::CallbackInfo& info) { + VALIDATE_ARGS_COUNT(info, 0, "getEncodeInferenceDuration()"); + return perf_utils::create_mean_std_pair(info.Env(), _metrics.get_encode_inference_duration()); +} + +Napi::Value ASRPerfMetricsWrapper::get_decode_inference_duration(const Napi::CallbackInfo& info) { + VALIDATE_ARGS_COUNT(info, 0, "getDecodeInferenceDuration()"); + return perf_utils::create_mean_std_pair(info.Env(), _metrics.get_decode_inference_duration()); +} + +Napi::Value ASRPerfMetricsWrapper::get_raw_metrics(const Napi::CallbackInfo& info) { + return BasePerfMetricsWrapper::get_raw_metrics(info); +} + +Napi::Value ASRPerfMetricsWrapper::get_asr_raw_metrics(const Napi::CallbackInfo& info) { + Napi::Object obj = Napi::Object::New(info.Env()); + obj.Set("featuresExtractionDurations", + cpp_to_js, Napi::Value>( + info.Env(), + get_ms(_metrics.asr_raw_metrics, &ov::genai::ASRRawPerfMetrics::features_extraction_durations))); + obj.Set("wordLevelTimestampsProcessingDurations", + cpp_to_js, Napi::Value>( + info.Env(), + get_ms(_metrics.asr_raw_metrics, + &ov::genai::ASRRawPerfMetrics::word_level_timestamps_processing_durations))); + obj.Set("encodeInferenceDurations", + cpp_to_js, Napi::Value>( + info.Env(), + get_ms(_metrics.asr_raw_metrics, &ov::genai::ASRRawPerfMetrics::encode_inference_durations))); + obj.Set("decodeInferenceDurations", + cpp_to_js, Napi::Value>( + info.Env(), + get_ms(_metrics.asr_raw_metrics, &ov::genai::ASRRawPerfMetrics::decode_inference_durations))); + return obj; +} diff --git a/src/js/src/asr_pipeline/pipeline_wrapper.cpp b/src/js/src/asr_pipeline/pipeline_wrapper.cpp new file mode 100644 index 0000000000..a2c2c505a3 --- /dev/null +++ b/src/js/src/asr_pipeline/pipeline_wrapper.cpp @@ -0,0 +1,265 @@ +// Copyright (C) 2023-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#include "include/asr_pipeline/pipeline_wrapper.hpp" + +#include +#include +#include +#include +#include + +#include "include/asr_pipeline/init_worker.hpp" +#include "include/helper.hpp" +#include "include/tokenizer.hpp" + +struct ASRTsfnContext { + ASRTsfnContext(std::vector raw_speech, + ov::AnyMap generation_config, + std::shared_ptr> is_generating) + : raw_speech(std::move(raw_speech)), + generation_config(std::move(generation_config)), + is_generating(is_generating) {} + ~ASRTsfnContext() = default; + + std::thread native_thread; + Napi::ThreadSafeFunction callback_tsfn; + std::optional streamer_tsfn; + + std::vector raw_speech; + ov::AnyMap generation_config; + std::shared_ptr> is_generating; + std::shared_ptr pipe = nullptr; +}; + +void asrPerformInferenceThread(ASRTsfnContext* context) { + auto report_error = [context](const std::string& message) { + auto status = context->callback_tsfn.BlockingCall([message](Napi::Env env, Napi::Function js_callback) { + try { + js_callback.Call( + {Napi::Error::New(env, "asrPerformInferenceThread error. " + message).Value(), env.Null()}); + } catch (const std::exception& err) { + std::cerr << "The callback failed when attempting to return an error from asrPerformInferenceThread. " + "Details:\n" + << err.what() << std::endl; + std::cerr << "Original error message:\n" << message << std::endl; + } + }); + if (status != napi_ok) { + std::cerr << "The BlockingCall failed with status " << status + << " when trying to return an error from asrPerformInferenceThread." << std::endl; + std::cerr << "Original error message:\n" << message << std::endl; + } + }; + auto finalize = [context]() { + context->callback_tsfn.Release(); + if (context->streamer_tsfn.has_value()) { + context->streamer_tsfn->Release(); + } + }; + std::vector streamer_exceptions; + ov::genai::ASRDecodedResults result; + + try { + ov::genai::StreamerVariant streamer_var = std::monostate(); + if (context->streamer_tsfn.has_value()) { + streamer_var = [context, &streamer_exceptions](std::string word) { + std::promise result_promise; + napi_status status = context->streamer_tsfn->BlockingCall( + [word, &result_promise, &streamer_exceptions](Napi::Env env, Napi::Function js_callback) { + try { + auto callback_result = js_callback.Call({Napi::String::New(env, word)}); + if (callback_result.IsNumber()) { + result_promise.set_value(static_cast( + callback_result.As().Int32Value())); + } else { + result_promise.set_value(ov::genai::StreamingStatus::RUNNING); + } + } catch (const std::exception& err) { + streamer_exceptions.push_back(err.what()); + result_promise.set_value(ov::genai::StreamingStatus::CANCEL); + } + }); + + if (status != napi_ok) { + streamer_exceptions.push_back("The streamer callback BlockingCall failed with status: " + + std::to_string(static_cast(status))); + return ov::genai::StreamingStatus::CANCEL; + } + return result_promise.get_future().get(); + }; + } + + // Move the raw speech into AudioInputs once. Passing the std::vector directly would + // trigger an implicit std::vector -> AudioInputs (std::variant) conversion that copies + // the whole audio buffer on every generate call. + ov::genai::AudioInputs audio_inputs{std::move(context->raw_speech)}; + if (context->streamer_tsfn.has_value()) { + auto config = context->pipe->get_generation_config(); + config.update_generation_config(context->generation_config); + result = context->pipe->generate(audio_inputs, config, streamer_var); + } else { + result = context->pipe->generate(audio_inputs, context->generation_config); + } + } catch (const std::exception& e) { + *context->is_generating = false; + report_error(e.what()); + finalize(); + return; + } + + *context->is_generating = false; + + try { + if (!streamer_exceptions.empty()) { + std::string combined_error = "Streamer exceptions occurred:\n"; + for (size_t i = 0; i < streamer_exceptions.size(); ++i) { + combined_error += "[" + std::to_string(i + 1) + "] " + streamer_exceptions[i] + "\n"; + } + report_error(combined_error); + } else { + std::shared_ptr final_result = + std::make_shared(std::move(result)); + std::shared_ptr final_callback_error = std::make_shared(); + + napi_status status = context->callback_tsfn.BlockingCall( + [final_result, final_callback_error](Napi::Env env, Napi::Function js_callback) { + try { + js_callback.Call({env.Null(), to_asr_decoded_result(env, *final_result)}); + } catch (const std::exception& err) { + *final_callback_error = "The final callback failed. Details:\n" + std::string(err.what()); + } + }); + if (status != napi_ok) { + report_error("The final BlockingCall failed with status " + std::to_string(static_cast(status))); + } else if (!final_callback_error->empty()) { + report_error(*final_callback_error); + } + } + } catch (const std::exception& e) { + report_error(e.what()); + } + finalize(); +} + +ASRPipelineWrapper::ASRPipelineWrapper(const Napi::CallbackInfo& info) : Napi::ObjectWrap(info) {} + +Napi::Function ASRPipelineWrapper::get_class(Napi::Env env) { + return DefineClass(env, + "ASRPipeline", + { + InstanceMethod("init", &ASRPipelineWrapper::init), + InstanceMethod("generate", &ASRPipelineWrapper::generate), + InstanceMethod("getTokenizer", &ASRPipelineWrapper::get_tokenizer), + InstanceMethod("getGenerationConfig", &ASRPipelineWrapper::get_generation_config), + InstanceMethod("setGenerationConfig", &ASRPipelineWrapper::set_generation_config), + }); +} + +Napi::Value ASRPipelineWrapper::init(const Napi::CallbackInfo& info) { + auto env = info.Env(); + try { + OPENVINO_ASSERT(!this->pipe, "Pipeline is already initialized"); + OPENVINO_ASSERT(!*this->is_initializing, "Pipeline is already initializing"); + *this->is_initializing = true; + + VALIDATE_ARGS_COUNT(info, 4, "init()"); + auto model_path = js_to_cpp(env, info[0]); + auto device = js_to_cpp(env, info[1]); + auto properties = js_to_cpp(env, info[2]); + OPENVINO_ASSERT(info[3].IsFunction(), "init callback is not a function"); + Napi::Function callback = info[3].As(); + + auto async_worker = new ASRInitWorker(callback, + this->pipe, + this->is_initializing, + std::move(model_path), + std::move(device), + std::move(properties)); + async_worker->Queue(); + } catch (const std::exception& ex) { + *this->is_initializing = false; + Napi::Error::New(env, ex.what()).ThrowAsJavaScriptException(); + } + + return env.Undefined(); +} + +Napi::Value ASRPipelineWrapper::generate(const Napi::CallbackInfo& info) { + auto env = info.Env(); + try { + OPENVINO_ASSERT(this->pipe, "ASRPipeline is not initialized"); + OPENVINO_ASSERT(!*this->is_generating, "Another generate is already in progress"); + *this->is_generating = true; + + VALIDATE_ARGS_COUNT(info, 4, "generate()"); + auto raw_speech = js_to_cpp>(env, info[0]); + auto generation_config = js_to_cpp(env, info[1]); + OPENVINO_ASSERT(info[3].IsFunction(), "generate callback is not a function"); + Napi::Function callback = info[3].As(); + + auto* context = new ASRTsfnContext(std::move(raw_speech), std::move(generation_config), this->is_generating); + context->pipe = this->pipe; + + context->callback_tsfn = + Napi::ThreadSafeFunction::New(env, callback, "ASR_generate_callback", 0, 1, [context](Napi::Env) { + context->native_thread.join(); + delete context; + }); + + if (info[2].IsFunction()) { + context->streamer_tsfn = + Napi::ThreadSafeFunction::New(env, info[2].As(), "ASR_generate_streamer", 0, 1); + } + + context->native_thread = std::thread(asrPerformInferenceThread, context); + } catch (const std::exception& ex) { + *this->is_generating = false; + Napi::Error::New(env, ex.what()).ThrowAsJavaScriptException(); + } + + return env.Undefined(); +} + +Napi::Value ASRPipelineWrapper::get_tokenizer(const Napi::CallbackInfo& info) { + auto env = info.Env(); + try { + OPENVINO_ASSERT(this->pipe, "ASRPipeline is not initialized"); + auto tokenizer = this->pipe->get_tokenizer(); + return TokenizerWrapper::wrap(env, tokenizer); + } catch (const std::exception& ex) { + Napi::Error::New(env, ex.what()).ThrowAsJavaScriptException(); + return env.Undefined(); + } +} + +Napi::Value ASRPipelineWrapper::get_generation_config(const Napi::CallbackInfo& info) { + auto env = info.Env(); + try { + OPENVINO_ASSERT(this->pipe, "ASRPipeline is not initialized"); + auto config = this->pipe->get_generation_config(); + return cpp_to_js(env, config); + } catch (const std::exception& ex) { + Napi::Error::New(env, ex.what()).ThrowAsJavaScriptException(); + return env.Undefined(); + } +} + +Napi::Value ASRPipelineWrapper::set_generation_config(const Napi::CallbackInfo& info) { + auto env = info.Env(); + try { + OPENVINO_ASSERT(this->pipe, "ASRPipeline is not initialized"); + VALIDATE_ARGS_COUNT(info, 1, "setGenerationConfig()"); + if (info[0].IsUndefined() || info[0].IsNull()) { + return env.Undefined(); + } + // Merge the provided fields onto the current config so partial updates keep required fields + // (e.g. stopping criteria) that ASRGenerationConfig::validate() expects. + auto config = this->pipe->get_generation_config(); + config.update_generation_config(js_to_cpp(env, info[0])); + this->pipe->set_generation_config(config); + } catch (const std::exception& ex) { + Napi::Error::New(env, ex.what()).ThrowAsJavaScriptException(); + } + return env.Undefined(); +} diff --git a/src/js/src/helper.cpp b/src/js/src/helper.cpp index 550de5304c..a66535b377 100644 --- a/src/js/src/helper.cpp +++ b/src/js/src/helper.cpp @@ -7,6 +7,7 @@ #include #include "include/addon.hpp" +#include "include/asr_pipeline/perf_metrics.hpp" #include "include/chat_history.hpp" #include "include/parser.hpp" #include "include/perf_metrics.hpp" @@ -24,6 +25,8 @@ constexpr const char* PARSERS_KEY = "parsers"; constexpr const char* STOP_CRITERIA_KEY = "stop_criteria"; constexpr const char* LANG_TO_ID_KEY = "lang_to_id"; constexpr const char* ALIGNMENT_HEADS_KEY = "alignment_heads"; +constexpr const char* SUPPRESS_TOKENS_KEY = "suppress_tokens"; +constexpr const char* BEGIN_SUPPRESS_TOKENS_KEY = "begin_suppress_tokens"; // Safe integer range for JS Number: -(2^53 - 1) .. (2^53 - 1). constexpr int64_t NAPI_NUMBER_MIN_INTEGER = -(1LL << 53) + 1; @@ -270,6 +273,10 @@ ov::AnyMap js_to_cpp(const Napi::Env& env, const Napi::Value& value) result_map[key_name] = js_to_cpp>(env, value_by_key); } else if (key_name == ALIGNMENT_HEADS_KEY) { result_map[key_name] = js_to_cpp>>(env, value_by_key); + } else if (key_name == SUPPRESS_TOKENS_KEY || key_name == BEGIN_SUPPRESS_TOKENS_KEY) { + // Force int64 element typing so empty arrays (used to clear suppression) don't + // become std::vector, which the generation config can't consume. + result_map[key_name] = js_to_cpp>(env, value_by_key); } else { result_map[key_name] = js_to_cpp(env, value_by_key); } @@ -835,6 +842,17 @@ ov::genai::WhisperPerfMetrics& unwrap(const Napi: return js_metrics->get_value(); } +template <> +ov::genai::ASRPerfMetrics& unwrap(const Napi::Env& env, const Napi::Value& value) { + const auto obj = value.As(); + const auto& prototype = env.GetInstanceData()->asr_perf_metrics; + OPENVINO_ASSERT(prototype, "Invalid pointer to prototype."); + OPENVINO_ASSERT(obj.InstanceOf(prototype.Value().As()), + "Passed argument is not of type ASRPerfMetrics"); + const auto js_metrics = Napi::ObjectWrap::Unwrap(obj); + return js_metrics->get_value(); +} + template <> ov::genai::SpeechGenerationPerfMetrics& unwrap(const Napi::Env& env, const Napi::Value& value) { @@ -1483,6 +1501,87 @@ Napi::Value cpp_to_js( return obj; } +template <> +Napi::Value cpp_to_js(const Napi::Env& env, + const ov::genai::ASRGenerationConfig& config) { + Napi::Object obj = cpp_to_js(env, config).As(); + if (config.language.has_value()) { + obj.Set("language", Napi::String::New(env, config.language.value())); + } else { + obj.Set("language", env.Undefined()); + } + if (config.task.has_value()) { + obj.Set("task", Napi::String::New(env, config.task.value())); + } else { + obj.Set("task", env.Undefined()); + } + obj.Set("return_timestamps", Napi::Boolean::New(env, config.return_timestamps)); + obj.Set("word_timestamps", Napi::Boolean::New(env, config.word_timestamps)); + obj.Set("decoder_start_token_id", cpp_to_js(env, config.decoder_start_token_id)); + obj.Set("pad_token_id", cpp_to_js(env, config.pad_token_id)); + obj.Set("translate_token_id", cpp_to_js(env, config.translate_token_id)); + obj.Set("transcribe_token_id", cpp_to_js(env, config.transcribe_token_id)); + obj.Set("prev_sot_token_id", cpp_to_js(env, config.prev_sot_token_id)); + obj.Set("no_timestamps_token_id", cpp_to_js(env, config.no_timestamps_token_id)); + obj.Set("max_initial_timestamp_index", cpp_to_js(env, config.max_initial_timestamp_index)); + obj.Set("is_multilingual", Napi::Boolean::New(env, config.is_multilingual)); + if (!config.lang_to_id.empty()) { + Napi::Object lang_to_id = Napi::Object::New(env); + for (const auto& [k, v] : config.lang_to_id) { + lang_to_id.Set(k, cpp_to_js(env, v)); + } + obj.Set("lang_to_id", lang_to_id); + } else { + obj.Set("lang_to_id", env.Undefined()); + } + if (!config.alignment_heads.empty()) { + Napi::Array arr = Napi::Array::New(env, config.alignment_heads.size()); + for (size_t i = 0; i < config.alignment_heads.size(); ++i) { + Napi::Array pair = Napi::Array::New(env, 2); + pair[0u] = cpp_to_js(env, config.alignment_heads[i].first); + pair[1u] = cpp_to_js(env, config.alignment_heads[i].second); + arr[static_cast(i)] = pair; + } + obj.Set("alignment_heads", arr); + } else { + obj.Set("alignment_heads", env.Undefined()); + } + if (config.initial_prompt.has_value()) { + obj.Set("initial_prompt", Napi::String::New(env, config.initial_prompt.value())); + } else { + obj.Set("initial_prompt", env.Undefined()); + } + if (config.hotwords.has_value()) { + obj.Set("hotwords", Napi::String::New(env, config.hotwords.value())); + } else { + obj.Set("hotwords", env.Undefined()); + } + if (config.context.has_value()) { + obj.Set("context", Napi::String::New(env, config.context.value())); + } else { + obj.Set("context", env.Undefined()); + } + if (!config.begin_suppress_tokens.empty()) { + Napi::Array arr = Napi::Array::New(env, config.begin_suppress_tokens.size()); + for (size_t i = 0; i < config.begin_suppress_tokens.size(); ++i) { + arr[static_cast(i)] = cpp_to_js(env, config.begin_suppress_tokens[i]); + } + obj.Set("begin_suppress_tokens", arr); + } else { + obj.Set("begin_suppress_tokens", env.Undefined()); + } + if (!config.suppress_tokens.empty()) { + Napi::Array arr = Napi::Array::New(env, config.suppress_tokens.size()); + for (size_t i = 0; i < config.suppress_tokens.size(); ++i) { + arr[static_cast(i)] = cpp_to_js(env, config.suppress_tokens[i]); + } + obj.Set("suppress_tokens", arr); + } else { + obj.Set("suppress_tokens", env.Undefined()); + } + return obj; +} + template <> Napi::Value cpp_to_js( const Napi::Env& env, @@ -1698,6 +1797,44 @@ Napi::Object to_whisper_decoded_result(const Napi::Env& env, const ov::genai::Wh return obj; } +Napi::Object to_asr_decoded_result(const Napi::Env& env, const ov::genai::ASRDecodedResults& results) { + Napi::Object obj = Napi::Object::New(env); + obj.Set("texts", cpp_to_js, Napi::Value>(env, results.texts)); + obj.Set("scores", cpp_to_js, Napi::Value>(env, results.scores)); + obj.Set("languages", cpp_to_js, Napi::Value>(env, results.languages)); + obj.Set("perfMetrics", ASRPerfMetricsWrapper::wrap(env, results.perf_metrics)); + + auto to_nested_chunks = [&env](const std::vector>& batched_chunks) { + Napi::Array batches = Napi::Array::New(env, batched_chunks.size()); + for (size_t b = 0; b < batched_chunks.size(); ++b) { + const auto& batch = batched_chunks[b]; + Napi::Array chunks = Napi::Array::New(env, batch.size()); + for (size_t i = 0; i < batch.size(); ++i) { + const auto& c = batch[i]; + Napi::Object chunk = Napi::Object::New(env); + chunk.Set("text", Napi::String::New(env, c.text)); + chunk.Set("startTs", cpp_to_js(env, c.start_ts)); + chunk.Set("endTs", cpp_to_js(env, c.end_ts)); + const size_t token_ids_size = c.token_ids.size(); + auto token_ids_buffer = Napi::ArrayBuffer::New(env, token_ids_size * sizeof(int64_t)); + std::memcpy(token_ids_buffer.Data(), c.token_ids.data(), token_ids_size * sizeof(int64_t)); + chunk.Set("tokenIds", Napi::BigInt64Array::New(env, token_ids_size, token_ids_buffer, 0)); + chunks[i] = chunk; + } + batches[b] = chunks; + } + return batches; + }; + + if (results.chunks.has_value()) { + obj.Set("chunks", to_nested_chunks(results.chunks.value())); + } + if (results.words.has_value()) { + obj.Set("words", to_nested_chunks(results.words.value())); + } + return obj; +} + Napi::Object to_text2speech_decoded_result(const Napi::Env& env, const ov::genai::Text2SpeechDecodedResults& results) { Napi::Object obj = Napi::Object::New(env); diff --git a/src/js/tests/asrPipeline.test.js b/src/js/tests/asrPipeline.test.js new file mode 100644 index 0000000000..2af3d127ba --- /dev/null +++ b/src/js/tests/asrPipeline.test.js @@ -0,0 +1,282 @@ +// Copyright (C) 2023-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +import { describe, it, before } from "node:test"; +import assert from "node:assert/strict"; +import { ASRPipeline, StreamingStatus } from "../dist/index.js"; +import { createTestRawSpeech } from "./utils.js"; +import { ASRPipeline as ASRPipelineClass } from "../dist/pipelines/asrPipeline.js"; + +// ASRPipeline dispatches to a model implementation (e.g. Whisper or Qwen3-ASR) based on the +// model's config.json. Whisper-specific capabilities (segment/word timestamps, `<|...|>` language +// tokens, task) are validated against a Whisper model, while a dedicated block validates Qwen3-ASR +// dispatch and its `context` option. +const { ASR_MODEL_PATH, WHISPER_MODEL_PATH } = process.env; + +if (!ASR_MODEL_PATH || !WHISPER_MODEL_PATH) { + throw new Error( + "Environment variables ASR_MODEL_PATH and WHISPER_MODEL_PATH must be set to the ASR model directories for tests.", + ); +} + +describe("ASRPipeline creation (Whisper backend)", () => { + it("ASRPipeline(modelPath, device) creates and initializes pipeline", async () => { + const pipeline = await ASRPipeline(WHISPER_MODEL_PATH, "CPU"); + assert.ok(pipeline); + assert.strictEqual(typeof pipeline.generate, "function"); + assert.strictEqual(typeof pipeline.getTokenizer, "function"); + assert.strictEqual(typeof pipeline.getGenerationConfig, "function"); + assert.strictEqual(typeof pipeline.setGenerationConfig, "function"); + }); + + it("ASRPipeline(modelPath, device, properties) accepts optional properties", async () => { + const pipeline = await ASRPipeline(WHISPER_MODEL_PATH, "CPU", {}); + assert.ok(pipeline); + }); +}); + +describe("ASRPipeline methods (Whisper backend)", () => { + let pipeline; + let rawSpeech; + + before(async () => { + pipeline = await ASRPipeline(WHISPER_MODEL_PATH, "CPU"); + rawSpeech = createTestRawSpeech(); + }); + + it("generate(rawSpeech) returns texts, scores, languages and perfMetrics", async () => { + const result = await pipeline.generate(rawSpeech); + assert.ok(Array.isArray(result.texts)); + assert.ok(Array.isArray(result.scores)); + assert.strictEqual(result.texts.length, result.scores.length); + assert.ok(Array.isArray(result.languages)); + assert.ok(result.perfMetrics); + assert.strictEqual(typeof result.perfMetrics.getLoadTime(), "number"); + const ttft = result.perfMetrics.getTTFT(); + assert.ok(ttft && typeof ttft === "object"); + assert.strictEqual(typeof ttft.mean, "number"); + assert.strictEqual(typeof ttft.std, "number"); + const tpot = result.perfMetrics.getTPOT(); + assert.ok(tpot && typeof tpot === "object"); + assert.strictEqual(typeof tpot.mean, "number"); + assert.strictEqual(typeof tpot.std, "number"); + }); + + it("generate(rawSpeech, options) accepts generationConfig", async () => { + const result = await pipeline.generate(rawSpeech, { + generationConfig: { language: "<|en|>", task: "transcribe" }, + }); + assert.ok(Array.isArray(result.texts)); + assert.ok(result.perfMetrics); + }); + + it("generate(rawSpeech, options) with streamer calls streamer with chunks", async () => { + const chunks = []; + const result = await pipeline.generate(rawSpeech, { + streamer: (chunk) => { + chunks.push(chunk); + assert.strictEqual(typeof chunk, "string"); + return StreamingStatus.RUNNING; + }, + }); + assert.ok(chunks.length > 0); + assert.strictEqual(chunks.join(""), result.texts[0]); + }); + + it("stream(rawSpeech) returns async iterator of chunks", async () => { + const chunks = []; + const stream = pipeline.stream(rawSpeech); + for await (const chunk of stream) { + chunks.push(chunk); + assert.strictEqual(typeof chunk, "string"); + } + assert.ok(Array.isArray(chunks)); + assert.ok(chunks.length > 0); + }); + + it("stream(rawSpeech, options) accepts generation config", async () => { + const chunks = []; + const generationConfig = { language: "<|en|>", task: "transcribe" }; + const stream = pipeline.stream(rawSpeech, { generationConfig }); + for await (const chunk of stream) { + chunks.push(chunk); + assert.strictEqual(typeof chunk, "string"); + } + assert.ok(Array.isArray(chunks)); + assert.ok(chunks.length > 0); + }); + + it("getTokenizer() returns tokenizer instance", () => { + const tokenizer = pipeline.getTokenizer(); + assert.ok(tokenizer); + assert.strictEqual(typeof tokenizer.encode, "function"); + assert.strictEqual(typeof tokenizer.decode, "function"); + }); + + it("getGenerationConfig() returns config object", () => { + const config = pipeline.getGenerationConfig(); + assert.ok(config && typeof config === "object"); + assert.strictEqual(typeof config.return_timestamps, "boolean"); + assert.strictEqual(typeof config.max_new_tokens, "bigint"); + }); + + it("setGenerationConfig(config) updates config", () => { + const newConfig = { initial_prompt: "hello", task: "transcribe" }; + pipeline.setGenerationConfig(newConfig); + const config = pipeline.getGenerationConfig(); + assert.strictEqual(config.initial_prompt, newConfig.initial_prompt); + assert.strictEqual(config.task, newConfig.task); + }); + + it("setGenerationConfig(config) accepts empty suppress token arrays", () => { + assert.doesNotThrow(() => { + pipeline.setGenerationConfig({ suppress_tokens: [], begin_suppress_tokens: [] }); + }); + }); + + it("throws when generate() is called before init()", async () => { + const uninitializedPipeline = new ASRPipelineClass("/nonexistent", "CPU"); + await assert.rejects( + uninitializedPipeline.generate(new Float32Array(100)), + /ASRPipeline is not initialized/, + ); + }); +}); + +describe("ASRPipeline with word_timestamps=true (Whisper backend)", () => { + let pipeline; + let rawSpeech; + + before(async () => { + // For Whisper-based models the word-level timestamps capability must be enabled + // via the constructor property; the request itself is made per generate call through + // the generation config (word_timestamps: true). + pipeline = await ASRPipeline(WHISPER_MODEL_PATH, "CPU", { word_timestamps: true }); + rawSpeech = createTestRawSpeech(); + }); + + it("setGenerationConfig({ word_timestamps: true }) is reflected by getGenerationConfig()", () => { + pipeline.setGenerationConfig({ word_timestamps: true }); + const config = pipeline.getGenerationConfig(); + assert.strictEqual(config.word_timestamps, true); + }); + + it("generate() with word_timestamps returns words with timestamps", async () => { + const result = await pipeline.generate(rawSpeech, { + generationConfig: { word_timestamps: true, language: "<|en|>", task: "transcribe" }, + }); + assert.ok(Array.isArray(result.words)); + // words are nested per input: words[inputIndex][wordIndex] + for (const perInput of result.words) { + assert.ok(Array.isArray(perInput)); + for (const w of perInput) { + assert.strictEqual(typeof w.text, "string"); + assert.strictEqual(typeof w.startTs, "number"); + assert.strictEqual(typeof w.endTs, "number"); + assert.ok(w.tokenIds instanceof BigInt64Array); + assert.ok(w.tokenIds.every((id) => typeof id === "bigint")); + } + } + }); + + it("generate() with return_timestamps returns chunks with timestamps", async () => { + const result = await pipeline.generate(rawSpeech, { + generationConfig: { return_timestamps: true }, + }); + assert.ok(result.chunks && result.chunks.length > 0); + for (const perInput of result.chunks) { + assert.ok(Array.isArray(perInput)); + for (const chunk of perInput) { + assert.strictEqual(typeof chunk.text, "string"); + assert.strictEqual(typeof chunk.startTs, "number"); + assert.strictEqual(typeof chunk.endTs, "number"); + } + } + }); +}); + +describe("ASRPerfMetrics (Whisper backend)", () => { + let pipeline; + let rawSpeech; + + before(async () => { + pipeline = await ASRPipeline(WHISPER_MODEL_PATH, "CPU"); + rawSpeech = createTestRawSpeech(); + }); + + it("result.perfMetrics has base getters and ASR-specific ones", async () => { + const result = await pipeline.generate(rawSpeech); + const pm = result.perfMetrics; + assert.ok(pm); + assert.strictEqual(typeof pm.getLoadTime(), "number"); + const ttft = pm.getTTFT(); + assert.ok(ttft && typeof ttft === "object"); + assert.strictEqual(typeof ttft.mean, "number"); + assert.strictEqual(typeof ttft.std, "number"); + const encodeDuration = pm.getEncodeInferenceDuration(); + assert.ok(encodeDuration && typeof encodeDuration === "object"); + assert.strictEqual(typeof encodeDuration.mean, "number"); + assert.strictEqual(typeof encodeDuration.std, "number"); + const decodeDuration = pm.getDecodeInferenceDuration(); + assert.ok(decodeDuration && typeof decodeDuration === "object"); + assert.strictEqual(typeof decodeDuration.mean, "number"); + assert.strictEqual(typeof decodeDuration.std, "number"); + const raw = pm.asrRawMetrics; + assert.ok(raw); + assert.ok(Array.isArray(raw.featuresExtractionDurations)); + assert.ok(Array.isArray(raw.wordLevelTimestampsProcessingDurations)); + assert.ok(Array.isArray(raw.encodeInferenceDurations)); + assert.ok(Array.isArray(raw.decodeInferenceDurations)); + }); + + it("getFeaturesExtractionDuration() returns MeanStdPair when available", async () => { + const result = await pipeline.generate(rawSpeech); + const pair = result.perfMetrics.getFeaturesExtractionDuration(); + assert.ok(pair && typeof pair === "object"); + assert.strictEqual(typeof pair.mean, "number"); + assert.strictEqual(typeof pair.std, "number"); + }); + + it("getWordLevelTimestampsProcessingDuration() returns MeanStdPair when available", async () => { + const result = await pipeline.generate(rawSpeech); + const pair = result.perfMetrics.getWordLevelTimestampsProcessingDuration(); + assert.ok(pair && typeof pair === "object"); + assert.strictEqual(typeof pair.mean, "number"); + assert.strictEqual(typeof pair.std, "number"); + }); +}); + +describe("ASRPipeline Qwen3-ASR dispatch", () => { + let pipeline; + let rawSpeech; + + before(async () => { + pipeline = await ASRPipeline(ASR_MODEL_PATH, "CPU"); + rawSpeech = createTestRawSpeech(); + }); + + it("generate(rawSpeech) returns texts, scores, languages and perfMetrics", async () => { + const result = await pipeline.generate(rawSpeech); + assert.ok(Array.isArray(result.texts)); + assert.ok(Array.isArray(result.scores)); + assert.strictEqual(result.texts.length, result.scores.length); + assert.ok(Array.isArray(result.languages)); + assert.ok(result.perfMetrics); + assert.strictEqual(typeof result.perfMetrics.getLoadTime(), "number"); + }); + + it("generate(rawSpeech, options) accepts the Qwen3-ASR context option", async () => { + const result = await pipeline.generate(rawSpeech, { + generationConfig: { context: "meeting transcript" }, + }); + assert.ok(Array.isArray(result.texts)); + assert.ok(result.perfMetrics); + }); + + it("getGenerationConfig() returns config object", () => { + const config = pipeline.getGenerationConfig(); + assert.ok(config && typeof config === "object"); + assert.strictEqual(typeof config.return_timestamps, "boolean"); + assert.strictEqual(typeof config.max_new_tokens, "bigint"); + }); +}); diff --git a/src/js/tests/setup.py b/src/js/tests/setup.py index 415aec13b5..ed40b0f553 100644 --- a/src/js/tests/setup.py +++ b/src/js/tests/setup.py @@ -43,6 +43,10 @@ "model_id": "cross-encoder/ms-marco-TinyBERT-L2-v2", "model_class": OVModelForSequenceClassification, }, + "ASR_MODEL": { + "model_id": "optimum-intel-internal-testing/tiny-random-qwen3-asr", + "model_class": OVModelForSpeechSeq2Seq, + }, "WHISPER_MODEL": { "model_id": "openai/whisper-tiny", "model_class": OVModelForSpeechSeq2Seq, diff --git a/tests/python_tests/samples/test_automatic_speech_recognition.py b/tests/python_tests/samples/test_automatic_speech_recognition.py index 7d8d7ccb6a..ddbe58fbf4 100644 --- a/tests/python_tests/samples/test_automatic_speech_recognition.py +++ b/tests/python_tests/samples/test_automatic_speech_recognition.py @@ -47,7 +47,7 @@ def test_sample_automatic_speech_recognition(self, convert_model, download_test_ c_result = run_sample(c_command) # Run JS sample - js_sample = SAMPLES_JS_DIR / "whisper_speech_recognition/whisper_speech_recognition.js" + js_sample = SAMPLES_JS_DIR / "automatic_speech_recognition/automatic_speech_recognition.js" js_command = ["node", js_sample, convert_model, download_test_content] js_result = run_sample(js_command)