[EXPERIMENT][WIP] Add PiperTTSImpl backend for Piper (VITS) text-to-speech voices

src/cpp/include/openvino/genai/speech_generation/speech_generation_config.hpp

@@ -56,6 +56,19 @@ class OPENVINO_GENAI_EXPORTS SpeechGenerationConfig : public GenerationConfig {
     // - unset: default to espeak-ng G2P fallback.
     std::optional<std::filesystem::path> phonemize_fallback_model_dir;
 
+    // ---------------------------------------------------------------------
+    // Piper-specific parameters
+    // ---------------------------------------------------------------------
+
+    // Stochastic duration/flow predictor noise scale; controls prosody variety.
+    float noise_scale = 0.667f;
+
+    // Multiplier applied to predicted phoneme durations; controls speaking rate.
+    float length_scale = 1.0f;
+
+    // Stochastic duration predictor noise weight.
+    float noise_w = 0.8f;
+
     void update_generation_config(const ov::AnyMap& config_map = {}) override;
 
     template <typename... Properties>
@@ -78,5 +91,9 @@ static constexpr ov::Property<std::string> speech_language{"language"};
 static constexpr ov::Property<uint32_t> max_phoneme_length{"max_phoneme_length"};
 static constexpr ov::Property<std::filesystem::path> phonemize_fallback_model_dir{"phonemize_fallback_model_dir"};
 
+static constexpr ov::Property<float> noise_scale{"noise_scale"};
+static constexpr ov::Property<float> length_scale{"length_scale"};
+static constexpr ov::Property<float> noise_w{"noise_w"};
+
 }  // namespace genai
 }  // namespace ov

src/cpp/include/openvino/genai/speech_generation/text2speech_pipeline.hpp

@@ -88,7 +88,7 @@ class OPENVINO_GENAI_EXPORTS Text2SpeechPipeline {
 
     /// @brief Get the expected speaker embedding shape for the loaded model.
     /// @return Shape describing the required tensor layout.
-    ///         SpeechT5: {1, 512}.  Kokoro: {510, 1, 256}
+    ///         SpeechT5: {1, 512}.  Kokoro: {510, 1, 256}.  Piper: {0} (single-speaker, no embedding used).
     ov::Shape get_speaker_embedding_shape() const;
 
 private:

src/cpp/src/speech_generation/misaki_cpp/include/misaki/fallbacks.hpp

@@ -43,4 +43,11 @@ class EspeakG2P {
   std::string library_path_;
 };
 
+/// Raw IPA phonemization without Misaki normalization (no diphthong tie-merging, no
+/// symbol remapping) — for consumers needing espeak-ng's individual IPA codepoints
+/// untouched (e.g. Piper TTS, which encodes each IPA codepoint as a separate token id).
+std::optional<std::string> raw_espeak_ipa_phonemize(const std::string& text,
+                                                     const std::string& voice_name,
+                                                     std::string library_path = {});
+
 } // namespace misaki

src/cpp/src/speech_generation/misaki_cpp/src/espeak_fallback.cpp

@@ -331,15 +331,19 @@ std::string normalize_espeak_to_misaki(std::string ps, bool british, const std::
 
 std::optional<std::string> raw_espeak_phonemize(EspeakApi &api,
                                                 const std::string &text,
-                                                const std::string &voice_name) {
+                                                const std::string &voice_name,
+                                                bool use_tie = true) {
   if (api.set_voice_by_name(voice_name.c_str()) != kEspeakOk) {
     return std::nullopt;
   }
 
   const void *text_ptr = static_cast<const void *>(text.c_str());
   std::string raw;
 
-  const int phoneme_mode = kEspeakPhonemesIpa | kEspeakPhonemesTie | (static_cast<int>('^') << 8);
+  int phoneme_mode = kEspeakPhonemesIpa;
+  if (use_tie) {
+    phoneme_mode |= kEspeakPhonemesTie | (static_cast<int>('^') << 8);
+  }
   while (text_ptr != nullptr) {
     const char *chunk = api.text_to_phonemes(&text_ptr, kEspeakCharsUtf8, phoneme_mode);
     if (!chunk) {
@@ -666,4 +670,18 @@ std::optional<std::string> EspeakG2P::backend_error() const {
   return state.error;
 }
 
+std::optional<std::string> raw_espeak_ipa_phonemize(const std::string &text,
+                                                    const std::string &voice_name,
+                                                    std::string library_path) {
+  static std::mutex api_mutex;
+  std::scoped_lock lock(api_mutex);
+
+  auto &state = get_cached_load_state(library_path);
+  if (!state.api.has_value()) {
+    return std::nullopt;
+  }
+
+  return raw_espeak_phonemize(*(state.api), text, voice_name, /*use_tie=*/false);
+}
+
 } // namespace misaki

src/cpp/src/speech_generation/piper_tts_model.cpp

@@ -0,0 +1,225 @@
+// Copyright (C) 2023-2026 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#include "piper_tts_model.hpp"
+
+#include <algorithm>
+#include <array>
+#include <chrono>
+#include <cstdlib>
+#include <fstream>
+#include <vector>
+
+#include <nlohmann/json.hpp>
+
+#include "openvino/genai/perf_metrics.hpp"
+#include "utils.hpp"
+#include "logger.hpp"
+
+#if OPENVINO_GENAI_HAS_MISAKI_CPP
+#include "misaki/fallbacks.hpp"
+#endif
+
+namespace ov {
+namespace genai {
+
+namespace {
+
+void set_default_property(ov::AnyMap& config, const std::string& key, const ov::Any& value) {
+    if (config.count(key) == 0) {
+        config.insert({key, value});
+    }
+}
+
+ov::CompiledModel compile_piper_model(ov::Core& core,
+                                      const std::filesystem::path& model_path,
+                                      const std::string& device,
+                                      const ov::AnyMap& properties) {
+    ov::AnyMap compile_properties = properties;
+    if (device.find("GPU") != std::string::npos) {
+        set_default_property(compile_properties, "INFERENCE_PRECISION_HINT", ov::element::f32);
+    }
+    return core.compile_model(model_path, device, compile_properties);
+}
+
+std::string find_input_name(const ov::CompiledModel& model, const std::string& substring, size_t fallback_index) {
+    for (const auto& input : model.inputs()) {
+        const std::string name = input.get_any_name();
+        if (name.find(substring) != std::string::npos) {
+            return name;
+        }
+    }
+    OPENVINO_ASSERT(model.inputs().size() > fallback_index,
+                    "Piper model is missing an expected input matching '",
+                    substring,
+                    "'");
+    return model.input(static_cast<int>(fallback_index)).get_any_name();
+}
+
+// Splits a UTF-8 string into individual Unicode codepoints, each returned as its own
+// UTF-8-encoded string. Piper's phoneme_id_map keys are single codepoints (including
+// combining diacritics, which attach to the preceding base character as a *separate*
+// sequence element rather than being composed).
+std::vector<std::string> split_into_codepoints(const std::string& utf8_text) {
+    std::vector<std::string> codepoints;
+    size_t index = 0;
+    while (index < utf8_text.size()) {
+        const unsigned char lead_byte = static_cast<unsigned char>(utf8_text[index]);
+        size_t char_length = 1;
+        if ((lead_byte & 0x80) == 0x00) {
+            char_length = 1;
+        } else if ((lead_byte & 0xE0) == 0xC0) {
+            char_length = 2;
+        } else if ((lead_byte & 0xF0) == 0xE0) {
+            char_length = 3;
+        } else if ((lead_byte & 0xF8) == 0xF0) {
+            char_length = 4;
+        }
+        char_length = std::min(char_length, utf8_text.size() - index);
+        codepoints.push_back(utf8_text.substr(index, char_length));
+        index += char_length;
+    }
+    return codepoints;
+}
+
+}  // namespace
+
+PiperTTSImpl::PiperTTSImpl(const std::filesystem::path& models_path,
+                           const std::string& device,
+                           const ov::AnyMap& properties) {
+    const auto load_start = std::chrono::steady_clock::now();
+    m_models_path = models_path;
+
+    const std::filesystem::path config_path = models_path / "config.json";
+    std::ifstream config_file(config_path);
+    OPENVINO_ASSERT(config_file.is_open(), "Failed to open Piper config at ", config_path);
+
+    nlohmann::json data = nlohmann::json::parse(config_file);
+    OPENVINO_ASSERT(data.contains("phoneme_id_map") && data["phoneme_id_map"].is_object(),
+                    "Piper config.json at ",
+                    config_path,
+                    " is missing the required 'phoneme_id_map' object");
+
+    for (auto it = data["phoneme_id_map"].begin(); it != data["phoneme_id_map"].end(); ++it) {
+        OPENVINO_ASSERT(it.value().is_array() && !it.value().empty(),
+                        "Piper phoneme_id_map entry '",
+                        it.key(),
+                        "' must be a non-empty array");
+        m_phoneme_id_map[it.key()] = it.value()[0].get<int64_t>();
+    }
+
+    if (data.contains("sample_rate") && data["sample_rate"].is_number_unsigned()) {
+        m_sample_rate = data["sample_rate"].get<uint32_t>();
+    }
+    if (data.contains("language") && data["language"].is_string()) {
+        m_espeak_voice = data["language"].get<std::string>();
+    }
+
+    ov::Core core = ov::genai::utils::singleton_core();
+    auto compiled = compile_piper_model(core, models_path / "openvino_model.xml", device, properties);
+    ov::genai::utils::print_compiled_model_properties(compiled, "piper model");
+
+    m_input_name = find_input_name(compiled, "input", 0);
+    m_input_lengths_name = find_input_name(compiled, "length", 1);
+    m_scales_name = find_input_name(compiled, "scales", 2);
+
+    m_request = compiled.create_infer_request();
+    save_load_time(load_start);
+}
+
+std::vector<int64_t> PiperTTSImpl::build_phoneme_id_sequence(const std::string& text) const {
+    auto id_of = [this](const std::string& symbol) -> int64_t {
+        const auto it = m_phoneme_id_map.find(symbol);
+        OPENVINO_ASSERT(it != m_phoneme_id_map.end(), "Piper phoneme_id_map has no entry for symbol '", symbol, "'");
+        return it->second;
+    };
+
+    const int64_t pad_id = id_of("_");
+    const int64_t bos_id = id_of("^");
+    const int64_t eos_id = id_of("$");
+
+    const std::vector<std::string> codepoints = split_into_codepoints(text);
+
+    std::vector<int64_t> sequence;
+    sequence.reserve(codepoints.size() * 2 + 3);
+    sequence.push_back(bos_id);
+    sequence.push_back(pad_id);
+    for (const std::string& codepoint : codepoints) {
+        const auto it = m_phoneme_id_map.find(codepoint);
+        if (it == m_phoneme_id_map.end()) {
+            // Skip phonemes absent from this voice's map (for example stray espeak
+            // language-switch artifacts) rather than aborting the whole utterance.
+            continue;
+        }
+        sequence.push_back(it->second);
+        sequence.push_back(pad_id);
+    }
+    sequence.push_back(eos_id);
+    return sequence;
+}
+
+Text2SpeechDecodedResults PiperTTSImpl::generate(const std::vector<std::string>& texts,
+                                                 const ov::Tensor& speaker_embedding,
+                                                 const SpeechGenerationConfig& generation_config) {
+    OPENVINO_ASSERT(!static_cast<bool>(speaker_embedding) || speaker_embedding.get_size() == 0,
+                    "Piper backend is single-speaker and does not accept a speaker_embedding tensor.");
+
+#if OPENVINO_GENAI_HAS_MISAKI_CPP
+    const char* library_hint = std::getenv("MISAKI_ESPEAK_LIBRARY");
+    const std::string library_path = library_hint ? std::string(library_hint) : std::string{};
+    const std::string espeak_voice = generation_config.language.empty() ? m_espeak_voice : generation_config.language;
+#endif
+
+    Text2SpeechDecodedResults result;
+    result.output_sample_rate = m_sample_rate;
+    const auto generation_start = std::chrono::steady_clock::now();
+
+    const std::array<float, 3> scales = {generation_config.noise_scale,
+                                         generation_config.length_scale,
+                                         generation_config.noise_w};
+
+    for (const std::string& text : texts) {
+#if OPENVINO_GENAI_HAS_MISAKI_CPP
+        const auto phonemized = misaki::raw_espeak_ipa_phonemize(text, espeak_voice, library_path);
+        OPENVINO_ASSERT(phonemized.has_value(),
+                        "Piper G2P failed to phonemize input text with espeak-ng voice '",
+                        espeak_voice,
+                        "'. Install espeak-ng and/or set MISAKI_ESPEAK_LIBRARY.");
+        const std::vector<int64_t> phoneme_ids = build_phoneme_id_sequence(*phonemized);
+#else
+        OPENVINO_THROW(
+            "Piper backend requires misaki-cpp for espeak-ng based G2P. Configure with ENABLE_MISAKI_CPP=ON and "
+            "provide misaki-cpp sources.");
+        const std::vector<int64_t> phoneme_ids;
+#endif
+
+        const int64_t input_length = static_cast<int64_t>(phoneme_ids.size());
+        ov::Tensor input_tensor(ov::element::i64, ov::Shape{1, phoneme_ids.size()}, const_cast<int64_t*>(phoneme_ids.data()));
+        ov::Tensor input_lengths_tensor(ov::element::i64, ov::Shape{1}, const_cast<int64_t*>(&input_length));
+        ov::Tensor scales_tensor(ov::element::f32, ov::Shape{3}, const_cast<float*>(scales.data()));
+
+        m_request.set_tensor(m_input_name, input_tensor);
+        m_request.set_tensor(m_input_lengths_name, input_lengths_tensor);
+        m_request.set_tensor(m_scales_name, scales_tensor);
+
+        m_request.infer();
+
+        const ov::Tensor waveform = m_request.get_output_tensor(0);
+        const float* waveform_data = waveform.data<const float>();
+        const size_t sample_count = waveform.get_size();
+
+        ov::Tensor speech(ov::element::f32, ov::Shape{sample_count});
+        std::copy(waveform_data, waveform_data + sample_count, speech.data<float>());
+        result.perf_metrics.num_generated_samples += speech.get_size();
+        result.speeches.push_back(speech);
+    }
+
+    result.perf_metrics.raw_metrics.generate_durations.emplace_back(
+        MicroSeconds(std::chrono::steady_clock::now() - generation_start));
+    result.perf_metrics.evaluate_statistics();
+    m_perf_metrics = result.perf_metrics;
+    return result;
+}
+
+}  // namespace genai
+}  // namespace ov

src/cpp/src/speech_generation/piper_tts_model.hpp

@@ -0,0 +1,48 @@
+// Copyright (C) 2023-2026 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include <cstdint>
+#include <filesystem>
+#include <string>
+#include <unordered_map>
+
+#include "openvino/genai/speech_generation/speech_generation_config.hpp"
+#include "text2speech_pipeline_impl.hpp"
+
+namespace ov {
+namespace genai {
+
+/// @brief Backend for Piper (VITS-family, single-graph, non-autoregressive) text-to-speech
+/// voices. Unlike Kokoro/SpeechT5 the model has no separate encoder/decoder split and
+/// performs a single forward pass to produce the full waveform. Each voice is single-speaker,
+/// so a non-empty speaker_embedding is rejected.
+class PiperTTSImpl : public Text2SpeechPipelineImpl {
+public:
+    PiperTTSImpl(const std::filesystem::path& models_path, const std::string& device, const ov::AnyMap& properties);
+
+    Text2SpeechDecodedResults generate(const std::vector<std::string>& texts,
+                                       const ov::Tensor& speaker_embedding,
+                                       const SpeechGenerationConfig& generation_config) override;
+
+    ov::Shape get_speaker_embedding_shape() const override {
+        // Piper voices bundled in this backend are single-speaker; no speaker embedding is used.
+        return ov::Shape{0};
+    }
+
+private:
+    std::vector<int64_t> build_phoneme_id_sequence(const std::string& text) const;
+
+    std::filesystem::path m_models_path;
+    ov::InferRequest m_request;
+    std::string m_input_name;
+    std::string m_input_lengths_name;
+    std::string m_scales_name;
+    std::unordered_map<std::string, int64_t> m_phoneme_id_map;
+    uint32_t m_sample_rate = 22050;
+    std::string m_espeak_voice = "en-us";
+};
+
+}  // namespace genai
+}  // namespace ov

src/cpp/src/speech_generation/speech_generation_config.cpp

@@ -30,6 +30,9 @@ SpeechGenerationConfig::SpeechGenerationConfig(const std::filesystem::path& json
         read_json_param(data, "language", language);
         read_json_param(data, "max_phoneme_length", max_phoneme_length);
         read_json_param(data, "phonemize_fallback_model_dir", phonemize_fallback_model_dir);
+        read_json_param(data, "noise_scale", noise_scale);
+        read_json_param(data, "length_scale", length_scale);
+        read_json_param(data, "noise_w", noise_w);
     }
     validate();
 }
@@ -44,6 +47,9 @@ void SpeechGenerationConfig::update_generation_config(const ov::AnyMap& config_m
     read_anymap_param(config_map, "language", language);
     read_anymap_param(config_map, "max_phoneme_length", max_phoneme_length);
     read_anymap_param(config_map, "phonemize_fallback_model_dir", phonemize_fallback_model_dir);
+    read_anymap_param(config_map, "noise_scale", noise_scale);
+    read_anymap_param(config_map, "length_scale", length_scale);
+    read_anymap_param(config_map, "noise_w", noise_w);
 
     GenerationConfig::update_generation_config(config_map);
 }
@@ -56,6 +62,9 @@ void SpeechGenerationConfig::validate() const {
     OPENVINO_ASSERT(max_phoneme_length > 0, "max_phoneme_length must be positive");
     OPENVINO_ASSERT(!phonemize_fallback_model_dir.has_value() || !phonemize_fallback_model_dir->empty(),
                     "phonemize_fallback_model_dir must be unset or a non-empty path");
+    OPENVINO_ASSERT(noise_scale >= 0.0f, "noise_scale must be non-negative");
+    OPENVINO_ASSERT(length_scale > 0.0f, "length_scale must be positive");
+    OPENVINO_ASSERT(noise_w >= 0.0f, "noise_w must be non-negative");
 }
 
 }  // namespace genai