From 2358aaac000369e1cd3790f9a5a25b3a10fc5a32 Mon Sep 17 00:00:00 2001 From: Omega-Intel Date: Wed, 8 Jul 2026 19:09:40 +0200 Subject: [PATCH] Add PiperTTSImpl backend for Piper (VITS) text-to-speech voices Adds a new PiperTTSImpl backend to Text2SpeechPipeline supporting Piper (rhasspy/piper-voices), a single-graph, non-autoregressive VITS-family TTS architecture with no separate encoder/decoder split. - New src/cpp/src/speech_generation/piper_tts_model.{hpp,cpp}: loads phoneme_id_map from config.json, builds the BOS/PAD/EOS phoneme-id sequence per Piper's convention, runs a single inference pass through the ONNX-derived OpenVINO IR (input/input_lengths/scales -> output), and rejects non-empty speaker_embedding (Piper voices are single-speaker). - Extended SpeechGenerationConfig with Piper-specific noise_scale, length_scale, and noise_w parameters (reusing the existing language field for the espeak-ng voice name). - Extended the Text2SpeechPipeline dispatcher (resolve_backend) with an additive Piper detection branch keyed on the architectures field or the presence of a phoneme_id_map object in config.json. - misaki_cpp: added an additive use_tie parameter (default true, preserving existing Kokoro behavior) to the internal raw_espeak_phonemize helper, and a new public misaki::raw_espeak_ipa_phonemize() function that reuses the existing espeak-ng loading/caching infrastructure to return untouched IPA codepoints (no Misaki diphthong-tie merging or symbol remapping), which Piper's per-codepoint phoneme_id_map requires. - New tests/python_tests/test_piper_pipeline.py and tests/python_tests/utils/piper_test_assets.py: a tiny ONNX/OpenVINO fixture matching Piper's exact I/O contract (no HF/optimum-cli export path exists for this architecture) plus unit tests for backend dispatch, waveform generation, speaker_embedding rejection, and configurable inference scales. Validated end-to-end against the real en_US-joe-medium and de_DE-thorsten-medium IRs (FP32/INT8/INT4) on CPU and GPU. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../speech_generation_config.hpp | 17 ++ .../text2speech_pipeline.hpp | 2 +- .../misaki_cpp/include/misaki/fallbacks.hpp | 7 + .../misaki_cpp/src/espeak_fallback.cpp | 22 +- .../src/speech_generation/piper_tts_model.cpp | 225 ++++++++++++++++++ .../src/speech_generation/piper_tts_model.hpp | 48 ++++ .../speech_generation_config.cpp | 9 + .../text2speech_pipeline.cpp | 21 ++ tests/python_tests/test_piper_pipeline.py | 125 ++++++++++ tests/python_tests/utils/piper_test_assets.py | 161 +++++++++++++ 10 files changed, 634 insertions(+), 3 deletions(-) create mode 100644 src/cpp/src/speech_generation/piper_tts_model.cpp create mode 100644 src/cpp/src/speech_generation/piper_tts_model.hpp create mode 100644 tests/python_tests/test_piper_pipeline.py create mode 100644 tests/python_tests/utils/piper_test_assets.py diff --git a/src/cpp/include/openvino/genai/speech_generation/speech_generation_config.hpp b/src/cpp/include/openvino/genai/speech_generation/speech_generation_config.hpp index 524d3eccc5..a772cbe51b 100644 --- a/src/cpp/include/openvino/genai/speech_generation/speech_generation_config.hpp +++ b/src/cpp/include/openvino/genai/speech_generation/speech_generation_config.hpp @@ -56,6 +56,19 @@ class OPENVINO_GENAI_EXPORTS SpeechGenerationConfig : public GenerationConfig { // - unset: default to espeak-ng G2P fallback. std::optional phonemize_fallback_model_dir; + // --------------------------------------------------------------------- + // Piper-specific parameters + // --------------------------------------------------------------------- + + // Stochastic duration/flow predictor noise scale; controls prosody variety. + float noise_scale = 0.667f; + + // Multiplier applied to predicted phoneme durations; controls speaking rate. + float length_scale = 1.0f; + + // Stochastic duration predictor noise weight. + float noise_w = 0.8f; + void update_generation_config(const ov::AnyMap& config_map = {}) override; template @@ -78,5 +91,9 @@ static constexpr ov::Property speech_language{"language"}; static constexpr ov::Property max_phoneme_length{"max_phoneme_length"}; static constexpr ov::Property phonemize_fallback_model_dir{"phonemize_fallback_model_dir"}; +static constexpr ov::Property noise_scale{"noise_scale"}; +static constexpr ov::Property length_scale{"length_scale"}; +static constexpr ov::Property noise_w{"noise_w"}; + } // namespace genai } // namespace ov diff --git a/src/cpp/include/openvino/genai/speech_generation/text2speech_pipeline.hpp b/src/cpp/include/openvino/genai/speech_generation/text2speech_pipeline.hpp index df54854fe2..1fda949565 100644 --- a/src/cpp/include/openvino/genai/speech_generation/text2speech_pipeline.hpp +++ b/src/cpp/include/openvino/genai/speech_generation/text2speech_pipeline.hpp @@ -88,7 +88,7 @@ class OPENVINO_GENAI_EXPORTS Text2SpeechPipeline { /// @brief Get the expected speaker embedding shape for the loaded model. /// @return Shape describing the required tensor layout. - /// SpeechT5: {1, 512}. Kokoro: {510, 1, 256} + /// SpeechT5: {1, 512}. Kokoro: {510, 1, 256}. Piper: {0} (single-speaker, no embedding used). ov::Shape get_speaker_embedding_shape() const; private: diff --git a/src/cpp/src/speech_generation/misaki_cpp/include/misaki/fallbacks.hpp b/src/cpp/src/speech_generation/misaki_cpp/include/misaki/fallbacks.hpp index ef0c71fb7e..df554157ed 100644 --- a/src/cpp/src/speech_generation/misaki_cpp/include/misaki/fallbacks.hpp +++ b/src/cpp/src/speech_generation/misaki_cpp/include/misaki/fallbacks.hpp @@ -43,4 +43,11 @@ class EspeakG2P { std::string library_path_; }; +/// Raw IPA phonemization without Misaki normalization (no diphthong tie-merging, no +/// symbol remapping) — for consumers needing espeak-ng's individual IPA codepoints +/// untouched (e.g. Piper TTS, which encodes each IPA codepoint as a separate token id). +std::optional raw_espeak_ipa_phonemize(const std::string& text, + const std::string& voice_name, + std::string library_path = {}); + } // namespace misaki diff --git a/src/cpp/src/speech_generation/misaki_cpp/src/espeak_fallback.cpp b/src/cpp/src/speech_generation/misaki_cpp/src/espeak_fallback.cpp index 61e583b6b1..75f2580f31 100644 --- a/src/cpp/src/speech_generation/misaki_cpp/src/espeak_fallback.cpp +++ b/src/cpp/src/speech_generation/misaki_cpp/src/espeak_fallback.cpp @@ -331,7 +331,8 @@ std::string normalize_espeak_to_misaki(std::string ps, bool british, const std:: std::optional raw_espeak_phonemize(EspeakApi &api, const std::string &text, - const std::string &voice_name) { + const std::string &voice_name, + bool use_tie = true) { if (api.set_voice_by_name(voice_name.c_str()) != kEspeakOk) { return std::nullopt; } @@ -339,7 +340,10 @@ std::optional raw_espeak_phonemize(EspeakApi &api, const void *text_ptr = static_cast(text.c_str()); std::string raw; - const int phoneme_mode = kEspeakPhonemesIpa | kEspeakPhonemesTie | (static_cast('^') << 8); + int phoneme_mode = kEspeakPhonemesIpa; + if (use_tie) { + phoneme_mode |= kEspeakPhonemesTie | (static_cast('^') << 8); + } while (text_ptr != nullptr) { const char *chunk = api.text_to_phonemes(&text_ptr, kEspeakCharsUtf8, phoneme_mode); if (!chunk) { @@ -666,4 +670,18 @@ std::optional EspeakG2P::backend_error() const { return state.error; } +std::optional raw_espeak_ipa_phonemize(const std::string &text, + const std::string &voice_name, + std::string library_path) { + static std::mutex api_mutex; + std::scoped_lock lock(api_mutex); + + auto &state = get_cached_load_state(library_path); + if (!state.api.has_value()) { + return std::nullopt; + } + + return raw_espeak_phonemize(*(state.api), text, voice_name, /*use_tie=*/false); +} + } // namespace misaki diff --git a/src/cpp/src/speech_generation/piper_tts_model.cpp b/src/cpp/src/speech_generation/piper_tts_model.cpp new file mode 100644 index 0000000000..eb3f88bc1a --- /dev/null +++ b/src/cpp/src/speech_generation/piper_tts_model.cpp @@ -0,0 +1,225 @@ +// Copyright (C) 2023-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#include "piper_tts_model.hpp" + +#include +#include +#include +#include +#include +#include + +#include + +#include "openvino/genai/perf_metrics.hpp" +#include "utils.hpp" +#include "logger.hpp" + +#if OPENVINO_GENAI_HAS_MISAKI_CPP +#include "misaki/fallbacks.hpp" +#endif + +namespace ov { +namespace genai { + +namespace { + +void set_default_property(ov::AnyMap& config, const std::string& key, const ov::Any& value) { + if (config.count(key) == 0) { + config.insert({key, value}); + } +} + +ov::CompiledModel compile_piper_model(ov::Core& core, + const std::filesystem::path& model_path, + const std::string& device, + const ov::AnyMap& properties) { + ov::AnyMap compile_properties = properties; + if (device.find("GPU") != std::string::npos) { + set_default_property(compile_properties, "INFERENCE_PRECISION_HINT", ov::element::f32); + } + return core.compile_model(model_path, device, compile_properties); +} + +std::string find_input_name(const ov::CompiledModel& model, const std::string& substring, size_t fallback_index) { + for (const auto& input : model.inputs()) { + const std::string name = input.get_any_name(); + if (name.find(substring) != std::string::npos) { + return name; + } + } + OPENVINO_ASSERT(model.inputs().size() > fallback_index, + "Piper model is missing an expected input matching '", + substring, + "'"); + return model.input(static_cast(fallback_index)).get_any_name(); +} + +// Splits a UTF-8 string into individual Unicode codepoints, each returned as its own +// UTF-8-encoded string. Piper's phoneme_id_map keys are single codepoints (including +// combining diacritics, which attach to the preceding base character as a *separate* +// sequence element rather than being composed). +std::vector split_into_codepoints(const std::string& utf8_text) { + std::vector codepoints; + size_t index = 0; + while (index < utf8_text.size()) { + const unsigned char lead_byte = static_cast(utf8_text[index]); + size_t char_length = 1; + if ((lead_byte & 0x80) == 0x00) { + char_length = 1; + } else if ((lead_byte & 0xE0) == 0xC0) { + char_length = 2; + } else if ((lead_byte & 0xF0) == 0xE0) { + char_length = 3; + } else if ((lead_byte & 0xF8) == 0xF0) { + char_length = 4; + } + char_length = std::min(char_length, utf8_text.size() - index); + codepoints.push_back(utf8_text.substr(index, char_length)); + index += char_length; + } + return codepoints; +} + +} // namespace + +PiperTTSImpl::PiperTTSImpl(const std::filesystem::path& models_path, + const std::string& device, + const ov::AnyMap& properties) { + const auto load_start = std::chrono::steady_clock::now(); + m_models_path = models_path; + + const std::filesystem::path config_path = models_path / "config.json"; + std::ifstream config_file(config_path); + OPENVINO_ASSERT(config_file.is_open(), "Failed to open Piper config at ", config_path); + + nlohmann::json data = nlohmann::json::parse(config_file); + OPENVINO_ASSERT(data.contains("phoneme_id_map") && data["phoneme_id_map"].is_object(), + "Piper config.json at ", + config_path, + " is missing the required 'phoneme_id_map' object"); + + for (auto it = data["phoneme_id_map"].begin(); it != data["phoneme_id_map"].end(); ++it) { + OPENVINO_ASSERT(it.value().is_array() && !it.value().empty(), + "Piper phoneme_id_map entry '", + it.key(), + "' must be a non-empty array"); + m_phoneme_id_map[it.key()] = it.value()[0].get(); + } + + if (data.contains("sample_rate") && data["sample_rate"].is_number_unsigned()) { + m_sample_rate = data["sample_rate"].get(); + } + if (data.contains("language") && data["language"].is_string()) { + m_espeak_voice = data["language"].get(); + } + + ov::Core core = ov::genai::utils::singleton_core(); + auto compiled = compile_piper_model(core, models_path / "openvino_model.xml", device, properties); + ov::genai::utils::print_compiled_model_properties(compiled, "piper model"); + + m_input_name = find_input_name(compiled, "input", 0); + m_input_lengths_name = find_input_name(compiled, "length", 1); + m_scales_name = find_input_name(compiled, "scales", 2); + + m_request = compiled.create_infer_request(); + save_load_time(load_start); +} + +std::vector PiperTTSImpl::build_phoneme_id_sequence(const std::string& text) const { + auto id_of = [this](const std::string& symbol) -> int64_t { + const auto it = m_phoneme_id_map.find(symbol); + OPENVINO_ASSERT(it != m_phoneme_id_map.end(), "Piper phoneme_id_map has no entry for symbol '", symbol, "'"); + return it->second; + }; + + const int64_t pad_id = id_of("_"); + const int64_t bos_id = id_of("^"); + const int64_t eos_id = id_of("$"); + + const std::vector codepoints = split_into_codepoints(text); + + std::vector sequence; + sequence.reserve(codepoints.size() * 2 + 3); + sequence.push_back(bos_id); + sequence.push_back(pad_id); + for (const std::string& codepoint : codepoints) { + const auto it = m_phoneme_id_map.find(codepoint); + if (it == m_phoneme_id_map.end()) { + // Skip phonemes absent from this voice's map (for example stray espeak + // language-switch artifacts) rather than aborting the whole utterance. + continue; + } + sequence.push_back(it->second); + sequence.push_back(pad_id); + } + sequence.push_back(eos_id); + return sequence; +} + +Text2SpeechDecodedResults PiperTTSImpl::generate(const std::vector& texts, + const ov::Tensor& speaker_embedding, + const SpeechGenerationConfig& generation_config) { + OPENVINO_ASSERT(!static_cast(speaker_embedding) || speaker_embedding.get_size() == 0, + "Piper backend is single-speaker and does not accept a speaker_embedding tensor."); + +#if OPENVINO_GENAI_HAS_MISAKI_CPP + const char* library_hint = std::getenv("MISAKI_ESPEAK_LIBRARY"); + const std::string library_path = library_hint ? std::string(library_hint) : std::string{}; + const std::string espeak_voice = generation_config.language.empty() ? m_espeak_voice : generation_config.language; +#endif + + Text2SpeechDecodedResults result; + result.output_sample_rate = m_sample_rate; + const auto generation_start = std::chrono::steady_clock::now(); + + const std::array scales = {generation_config.noise_scale, + generation_config.length_scale, + generation_config.noise_w}; + + for (const std::string& text : texts) { +#if OPENVINO_GENAI_HAS_MISAKI_CPP + const auto phonemized = misaki::raw_espeak_ipa_phonemize(text, espeak_voice, library_path); + OPENVINO_ASSERT(phonemized.has_value(), + "Piper G2P failed to phonemize input text with espeak-ng voice '", + espeak_voice, + "'. Install espeak-ng and/or set MISAKI_ESPEAK_LIBRARY."); + const std::vector phoneme_ids = build_phoneme_id_sequence(*phonemized); +#else + OPENVINO_THROW( + "Piper backend requires misaki-cpp for espeak-ng based G2P. Configure with ENABLE_MISAKI_CPP=ON and " + "provide misaki-cpp sources."); + const std::vector phoneme_ids; +#endif + + const int64_t input_length = static_cast(phoneme_ids.size()); + ov::Tensor input_tensor(ov::element::i64, ov::Shape{1, phoneme_ids.size()}, const_cast(phoneme_ids.data())); + ov::Tensor input_lengths_tensor(ov::element::i64, ov::Shape{1}, const_cast(&input_length)); + ov::Tensor scales_tensor(ov::element::f32, ov::Shape{3}, const_cast(scales.data())); + + m_request.set_tensor(m_input_name, input_tensor); + m_request.set_tensor(m_input_lengths_name, input_lengths_tensor); + m_request.set_tensor(m_scales_name, scales_tensor); + + m_request.infer(); + + const ov::Tensor waveform = m_request.get_output_tensor(0); + const float* waveform_data = waveform.data(); + const size_t sample_count = waveform.get_size(); + + ov::Tensor speech(ov::element::f32, ov::Shape{sample_count}); + std::copy(waveform_data, waveform_data + sample_count, speech.data()); + result.perf_metrics.num_generated_samples += speech.get_size(); + result.speeches.push_back(speech); + } + + result.perf_metrics.raw_metrics.generate_durations.emplace_back( + MicroSeconds(std::chrono::steady_clock::now() - generation_start)); + result.perf_metrics.evaluate_statistics(); + m_perf_metrics = result.perf_metrics; + return result; +} + +} // namespace genai +} // namespace ov diff --git a/src/cpp/src/speech_generation/piper_tts_model.hpp b/src/cpp/src/speech_generation/piper_tts_model.hpp new file mode 100644 index 0000000000..a47ac88ed4 --- /dev/null +++ b/src/cpp/src/speech_generation/piper_tts_model.hpp @@ -0,0 +1,48 @@ +// Copyright (C) 2023-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include + +#include "openvino/genai/speech_generation/speech_generation_config.hpp" +#include "text2speech_pipeline_impl.hpp" + +namespace ov { +namespace genai { + +/// @brief Backend for Piper (VITS-family, single-graph, non-autoregressive) text-to-speech +/// voices. Unlike Kokoro/SpeechT5 the model has no separate encoder/decoder split and +/// performs a single forward pass to produce the full waveform. Each voice is single-speaker, +/// so a non-empty speaker_embedding is rejected. +class PiperTTSImpl : public Text2SpeechPipelineImpl { +public: + PiperTTSImpl(const std::filesystem::path& models_path, const std::string& device, const ov::AnyMap& properties); + + Text2SpeechDecodedResults generate(const std::vector& texts, + const ov::Tensor& speaker_embedding, + const SpeechGenerationConfig& generation_config) override; + + ov::Shape get_speaker_embedding_shape() const override { + // Piper voices bundled in this backend are single-speaker; no speaker embedding is used. + return ov::Shape{0}; + } + +private: + std::vector build_phoneme_id_sequence(const std::string& text) const; + + std::filesystem::path m_models_path; + ov::InferRequest m_request; + std::string m_input_name; + std::string m_input_lengths_name; + std::string m_scales_name; + std::unordered_map m_phoneme_id_map; + uint32_t m_sample_rate = 22050; + std::string m_espeak_voice = "en-us"; +}; + +} // namespace genai +} // namespace ov diff --git a/src/cpp/src/speech_generation/speech_generation_config.cpp b/src/cpp/src/speech_generation/speech_generation_config.cpp index 3dea310c83..89ffc07980 100644 --- a/src/cpp/src/speech_generation/speech_generation_config.cpp +++ b/src/cpp/src/speech_generation/speech_generation_config.cpp @@ -30,6 +30,9 @@ SpeechGenerationConfig::SpeechGenerationConfig(const std::filesystem::path& json read_json_param(data, "language", language); read_json_param(data, "max_phoneme_length", max_phoneme_length); read_json_param(data, "phonemize_fallback_model_dir", phonemize_fallback_model_dir); + read_json_param(data, "noise_scale", noise_scale); + read_json_param(data, "length_scale", length_scale); + read_json_param(data, "noise_w", noise_w); } validate(); } @@ -44,6 +47,9 @@ void SpeechGenerationConfig::update_generation_config(const ov::AnyMap& config_m read_anymap_param(config_map, "language", language); read_anymap_param(config_map, "max_phoneme_length", max_phoneme_length); read_anymap_param(config_map, "phonemize_fallback_model_dir", phonemize_fallback_model_dir); + read_anymap_param(config_map, "noise_scale", noise_scale); + read_anymap_param(config_map, "length_scale", length_scale); + read_anymap_param(config_map, "noise_w", noise_w); GenerationConfig::update_generation_config(config_map); } @@ -56,6 +62,9 @@ void SpeechGenerationConfig::validate() const { OPENVINO_ASSERT(max_phoneme_length > 0, "max_phoneme_length must be positive"); OPENVINO_ASSERT(!phonemize_fallback_model_dir.has_value() || !phonemize_fallback_model_dir->empty(), "phonemize_fallback_model_dir must be unset or a non-empty path"); + OPENVINO_ASSERT(noise_scale >= 0.0f, "noise_scale must be non-negative"); + OPENVINO_ASSERT(length_scale > 0.0f, "length_scale must be positive"); + OPENVINO_ASSERT(noise_w >= 0.0f, "noise_w must be non-negative"); } } // namespace genai diff --git a/src/cpp/src/speech_generation/text2speech_pipeline.cpp b/src/cpp/src/speech_generation/text2speech_pipeline.cpp index 90d2330dea..9d0020fa56 100644 --- a/src/cpp/src/speech_generation/text2speech_pipeline.cpp +++ b/src/cpp/src/speech_generation/text2speech_pipeline.cpp @@ -11,6 +11,7 @@ #include "json_utils.hpp" #include "kokoro_tts_model.hpp" #include "openvino/genai/speech_generation/speech_generation_config.hpp" +#include "piper_tts_model.hpp" #include "speecht5_tts_model.hpp" #include "utils.hpp" @@ -36,14 +37,32 @@ const std::string get_class_name(const std::filesystem::path& root_dir) { enum class SpeechBackend { SpeechT5, Kokoro, + Piper, }; +bool has_phoneme_id_map(const std::filesystem::path& root_dir) { + const std::filesystem::path config_json_path = root_dir / "config.json"; + if (!std::filesystem::exists(config_json_path)) { + return false; + } + std::ifstream config_file(config_json_path); + if (!config_file.is_open()) { + return false; + } + const nlohmann::json config = nlohmann::json::parse(config_file, nullptr, false); + return !config.is_discarded() && config.contains("phoneme_id_map") && config["phoneme_id_map"].is_object(); +} + SpeechBackend resolve_backend(const std::filesystem::path& root_dir, const std::string& class_name) { if (class_name == "SpeechT5ForTextToSpeech") { return SpeechBackend::SpeechT5; } + if (class_name.find("Piper") != std::string::npos || has_phoneme_id_map(root_dir)) { + return SpeechBackend::Piper; + } + const bool has_openvino_model = std::filesystem::exists(root_dir / "openvino_model.xml"); const bool architecture_mentions_kokoro = class_name.find("Kokoro") != std::string::npos; @@ -81,6 +100,8 @@ Text2SpeechPipeline::Text2SpeechPipeline(const std::filesystem::path& root_dir, if (backend == SpeechBackend::SpeechT5) { auto tokenizer = ov::genai::Tokenizer(root_dir); m_impl = std::make_shared(root_dir, device, properties, tokenizer); + } else if (backend == SpeechBackend::Piper) { + m_impl = std::make_shared(root_dir, device, properties); } else { m_impl = std::make_shared(root_dir, device, properties); } diff --git a/tests/python_tests/test_piper_pipeline.py b/tests/python_tests/test_piper_pipeline.py new file mode 100644 index 0000000000..4a5365f0f0 --- /dev/null +++ b/tests/python_tests/test_piper_pipeline.py @@ -0,0 +1,125 @@ +# Copyright (C) 2025-2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +""" +Tests for the Piper (VITS-family) Text-to-Speech pipeline. + +Piper is a single-graph, non-autoregressive TTS model with no HuggingFace/optimum-cli +export path (unlike Kokoro/SpeechT5), so these tests use a dynamically-generated tiny +ONNX/OpenVINO fixture (see ``utils.piper_test_assets``) matching Piper's exact I/O +contract, rather than an ``optimum.intel`` model. +""" + +import logging +import os +from pathlib import Path + +import numpy as np +import pytest + +import openvino_genai as ov_genai +from utils.constants import get_ov_cache_converted_models_dir +from utils.piper_test_assets import prepare_tiny_piper_ov_path + +logger = logging.getLogger(__name__) + + +def _configure_espeak_from_venv() -> None: + """ + Point the C++ misaki espeak fallback (reused by PiperTTSImpl for raw IPA + phonemization) at the shared library/data bundled with the ``espeakng_loader`` + Python package, for CI machines without a system-wide espeak-ng install. + """ + try: + import espeakng_loader # type: ignore[import] + except ImportError: + logger.debug("espeakng_loader not available; skipping espeak env setup") + return + + lib_path = espeakng_loader.get_library_path() + data_path = espeakng_loader.get_data_path() + + if "MISAKI_ESPEAK_LIBRARY" not in os.environ: + os.environ["MISAKI_ESPEAK_LIBRARY"] = lib_path + if "ESPEAK_DATA_PATH" not in os.environ: + os.environ["ESPEAK_DATA_PATH"] = data_path + + +_configure_espeak_from_venv() + +SAMPLE_RATE = 22050 + + +@pytest.fixture(scope="module") +def tiny_piper_ov_path() -> Path: + """Fixture that generates a tiny random Piper OpenVINO model directory for testing.""" + models_dir = get_ov_cache_converted_models_dir() + return prepare_tiny_piper_ov_path(models_dir) + + +@pytest.mark.speech_generation +class TestPiperPipeline: + """Test suite for the Piper text-to-speech pipeline.""" + + def test_dispatcher_detects_piper_backend(self, tiny_piper_ov_path: Path): + """ + The Text2SpeechPipeline dispatcher must route a directory whose config.json + contains a ``phoneme_id_map`` object to the Piper backend without raising. + """ + pipe = ov_genai.Text2SpeechPipeline(str(tiny_piper_ov_path), "CPU") + assert tuple(pipe.get_speaker_embedding_shape()) == (0,) + + def test_genai_piper_generate_produces_valid_waveform(self, tiny_piper_ov_path: Path): + """Smoke test: a finite, non-empty 1D waveform must be produced for English text.""" + pipe = ov_genai.Text2SpeechPipeline(str(tiny_piper_ov_path), "CPU") + result = pipe.generate("hello test", language="en-us") + + speech = result.speeches[0] + speech_array = np.array(speech.data, dtype=np.float32).reshape(-1) + + assert result.output_sample_rate == SAMPLE_RATE + assert speech_array.ndim == 1 + assert speech_array.size > 0 + assert np.isfinite(speech_array).all() + + def test_piper_rejects_non_empty_speaker_embedding(self, tiny_piper_ov_path: Path): + """ + Piper voices are single-speaker; passing a non-empty speaker_embedding tensor + must raise rather than being silently ignored. + """ + import openvino as ov + + pipe = ov_genai.Text2SpeechPipeline(str(tiny_piper_ov_path), "CPU") + bad_embedding = ov.Tensor(np.zeros((1, 256), dtype=np.float32)) + + with pytest.raises(RuntimeError): + pipe.generate("hello test", bad_embedding, language="en-us") + + def test_piper_output_length_scales_with_text_length(self, tiny_piper_ov_path: Path): + """ + Longer input text should map to a longer phoneme-id sequence, and therefore to a + longer waveform for this fixture's length-proportional structural graph. + """ + pipe = ov_genai.Text2SpeechPipeline(str(tiny_piper_ov_path), "CPU") + + short_result = pipe.generate("ab", language="en-us") + long_result = pipe.generate("ab ab ab ab ab", language="en-us") + + short_len = np.array(short_result.speeches[0].data).size + long_len = np.array(long_result.speeches[0].data).size + + assert long_len > short_len + + def test_piper_scales_are_configurable(self, tiny_piper_ov_path: Path): + """noise_scale/length_scale/noise_w must be accepted as generation properties.""" + pipe = ov_genai.Text2SpeechPipeline(str(tiny_piper_ov_path), "CPU") + result = pipe.generate( + "hello test", + language="en-us", + noise_scale=0.5, + length_scale=1.2, + noise_w=0.6, + ) + speech_array = np.array(result.speeches[0].data, dtype=np.float32).reshape(-1) + assert speech_array.size > 0 + assert np.isfinite(speech_array).all() diff --git a/tests/python_tests/utils/piper_test_assets.py b/tests/python_tests/utils/piper_test_assets.py new file mode 100644 index 0000000000..ff40898b02 --- /dev/null +++ b/tests/python_tests/utils/piper_test_assets.py @@ -0,0 +1,161 @@ +# Copyright (C) 2025-2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +""" +Tiny-random Piper (VITS-family) ONNX/OpenVINO fixture generator for fast unit testing. + +Piper voices are distributed as a single raw ONNX graph (opset 15, ai.onnx domain only, +no custom/contrib ops) with a fixed I/O contract: + + Input "input" int64 [1, num_phoneme_ids] + Input "input_lengths" int64 [1] + Input "scales" float32 [3] == [noise_scale, length_scale, noise_w] + Output "output" float32 [1, time, 1, 1] + +Unlike Kokoro/SpeechT5 there is no HuggingFace/optimum-cli export path for this +architecture (it is not a Transformers model), so this fixture builds a tiny ONNX graph +directly with `onnx.helper` instead of exporting from a PyTorch checkpoint. The graph is a +structural stand-in only (embedding lookup -> linear upsample to a token-length-dependent +waveform); it does not reproduce Piper's real VITS acoustic modelling, only its I/O +contract, so tests built on it validate GenAI plumbing/parsing, not audio quality. +""" + +import io +import json +import logging +import shutil +from pathlib import Path + +import numpy as np +import onnx +import openvino as ov +from onnx import TensorProto, helper, numpy_helper + +from .atomic_download import AtomicDownloadManager + +logger = logging.getLogger(__name__) + +TINY_PIPER_SEED = 2024 +TINY_PIPER_SAMPLE_RATE = 22050 +TINY_PIPER_HIDDEN_DIM = 8 +# Number of upsampled audio samples generated per input phoneme-id, so output length scales +# with (and reveals bugs in) the phoneme-id sequence building logic under test. +TINY_PIPER_SAMPLES_PER_TOKEN = 256 + +# Minimal, single-Unicode-codepoint phoneme_id_map covering BOS/PAD/EOS plus a handful of +# IPA-like letters/punctuation, following the real Piper sidecar convention: +# PAD = phoneme_id_map["_"][0] (0), BOS = phoneme_id_map["^"][0] (1), EOS = phoneme_id_map["$"][0] (2) +TINY_PHONEME_ID_MAP = { + "_": [0], + "^": [1], + "$": [2], + " ": [3], + ".": [4], + "a": [5], + "b": [6], + "e": [7], + "h": [8], + "l": [9], + "o": [10], + "t": [11], + "s": [12], +} +TINY_VOCAB_SIZE = max(ids[0] for ids in TINY_PHONEME_ID_MAP.values()) + 1 + + +def _build_tiny_piper_onnx_model() -> onnx.ModelProto: + """ + Build a tiny ONNX graph matching Piper's exact I/O contract. + + Graph: Gather(embedding, input) -> ReduceMean(axis=1) -> Tile by + (input_lengths[0] * TINY_PIPER_SAMPLES_PER_TOKEN) -> reshape to [1, time, 1, 1]. + `scales` is consumed (added elementwise, scaled by a tiny constant) purely to exercise + the input being wired up correctly; it does not meaningfully affect output shape. + """ + rng = np.random.default_rng(TINY_PIPER_SEED) + embedding = rng.uniform(-0.1, 0.1, size=(TINY_VOCAB_SIZE, TINY_PIPER_HIDDEN_DIM)).astype(np.float32) + + input_ids = helper.make_tensor_value_info("input", TensorProto.INT64, [1, None]) + input_lengths = helper.make_tensor_value_info("input_lengths", TensorProto.INT64, [1]) + scales = helper.make_tensor_value_info("scales", TensorProto.FLOAT, [3]) + output = helper.make_tensor_value_info("output", TensorProto.FLOAT, [1, None, 1, 1]) + + embedding_initializer = numpy_helper.from_array(embedding, name="embedding_table") + + nodes = [ + helper.make_node("Gather", ["embedding_table", "input"], ["gathered"], axis=0), + helper.make_node("ReduceMean", ["gathered"], ["pooled"], axes=[1, 2], keepdims=0), + helper.make_node("ReduceSum", ["scales"], ["scale_sum"], keepdims=1), + helper.make_node("ReduceMean", ["pooled"], ["pooled_scalar"], axes=[0], keepdims=0), + helper.make_node("Add", ["pooled_scalar", "scale_sum"], ["mixed_scalar"]), + helper.make_node( + "Mul", + ["input_lengths", numpy_helper.from_array(np.array([TINY_PIPER_SAMPLES_PER_TOKEN], dtype=np.int64), "spt").name], + ["time_len"], + ), + helper.make_node("Reshape", ["time_len", numpy_helper.from_array(np.array([1], dtype=np.int64), "one_shape").name], ["time_len_1d"]), + helper.make_node("Expand", ["mixed_scalar", "time_len_1d"], ["expanded"]), + helper.make_node( + "Reshape", + ["expanded", numpy_helper.from_array(np.array([1, -1, 1, 1], dtype=np.int64), "out_shape").name], + ["output"], + ), + ] + + graph = helper.make_graph( + nodes, + "tiny_piper_vits", + [input_ids, input_lengths, scales], + [output], + initializer=[ + embedding_initializer, + numpy_helper.from_array(np.array([TINY_PIPER_SAMPLES_PER_TOKEN], dtype=np.int64), "spt"), + numpy_helper.from_array(np.array([1], dtype=np.int64), "one_shape"), + numpy_helper.from_array(np.array([1, -1, 1, 1], dtype=np.int64), "out_shape"), + ], + ) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 15)]) + onnx.checker.check_model(model) + return model + + +def generate_tiny_piper_model(output_dir: Path, voice_name: str = "tiny-random-piper", espeak_voice: str = "en-us") -> Path: + """Generate a tiny random Piper genai model directory (config.json + OpenVINO IR).""" + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + onnx_model = _build_tiny_piper_onnx_model() + onnx_bytes = io.BytesIO(onnx_model.SerializeToString()) + ov_model = ov.convert_model(onnx_bytes) + ov.save_model(ov_model, str(output_dir / "openvino_model.xml")) + + config = { + "architectures": ["PiperForTextToSpeech"], + "phoneme_id_map": TINY_PHONEME_ID_MAP, + "sample_rate": TINY_PIPER_SAMPLE_RATE, + "language": espeak_voice, + "noise_scale": 0.667, + "length_scale": 1.0, + "noise_w": 0.8, + } + with open(output_dir / "config.json", "w", encoding="utf-8") as config_file: + json.dump(config, config_file, indent=2) + + logger.info("Generated tiny Piper model at %s", output_dir) + return output_dir + + +def prepare_tiny_piper_ov_path(converted_models_dir: Path) -> Path: + """Prepare a tiny random Piper OpenVINO model directory in cache and return its path.""" + model_path = Path(converted_models_dir) / "tiny-random-piper-ov" + manager = AtomicDownloadManager(model_path) + manager.execute(lambda temp_path: generate_tiny_piper_model(temp_path / "model")) + + nested = model_path / "model" + if nested.exists() and (nested / "config.json").exists(): + if not (model_path / "config.json").exists(): + for item in nested.iterdir(): + shutil.move(str(item), str(model_path / item.name)) + shutil.rmtree(nested, ignore_errors=True) + + return model_path