feat: cloudsway tts extension

ai_agents/agents/ten_packages/extension/cloudsway_tts_python/README.md

@@ -0,0 +1,29 @@
+# cloudsway_tts_python
+
+<!-- brief introduction for the extension -->
+
+## Features
+
+<!-- main features introduction -->
+
+- xxx feature
+
+## API
+
+Refer to `api` definition in [manifest.json] and default values in [property.json](property.json).
+
+<!-- Additional API.md can be referred to if extra introduction needed -->
+
+## Development
+
+### Build
+
+<!-- build dependencies and steps -->
+
+### Unit test
+
+<!-- how to do unit test for the extension -->
+
+## Misc
+
+<!-- others if applicable -->

ai_agents/agents/ten_packages/extension/cloudsway_tts_python/__init__.py

@@ -0,0 +1,6 @@
+#
+# This file is part of TEN Framework, an open source project.
+# Licensed under the Apache License, Version 2.0.
+# See the LICENSE file for more information.
+#
+from . import addon

ai_agents/agents/ten_packages/extension/cloudsway_tts_python/addon.py

@@ -0,0 +1,20 @@
+#
+# This file is part of TEN Framework, an open source project.
+# Licensed under the Apache License, Version 2.0.
+# See the LICENSE file for more information.
+#
+from ten_runtime import (
+    Addon,
+    register_addon_as_extension,
+    TenEnv,
+)
+
+
+@register_addon_as_extension("cloudsway_tts_python")
+class CloudswayTTSExtensionAddon(Addon):
+
+    def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None:
+        from .extension import CloudswayTTSExtension
+
+        ten_env.log_info("CloudswayTTSExtensionAddon on_create_instance")
+        ten_env.on_create_instance_done(CloudswayTTSExtension(name), context)

ai_agents/agents/ten_packages/extension/cloudsway_tts_python/cloudsway_tts.py

@@ -0,0 +1,84 @@
+import asyncio
+from dataclasses import dataclass
+import aiohttp
+from datetime import datetime
+from typing import AsyncIterator
+
+from ten_runtime.async_ten_env import AsyncTenEnv
+from ten_ai_base.config import BaseConfig
+
+
+@dataclass
+class CloudswayTTSConfig(BaseConfig):
+    api_key: str = ""
+    model: str = "cloudsway_tts"
+    voice_id: str = "Luna_normal_1"
+    sample_rate: int = 32000
+    url: str = "http://0.0.0.0:9880/tts/stream"
+    language: str = "en"
+
+
+class CloudswayTTS:
+    def __init__(self, config: CloudswayTTSConfig):
+        self.config = config
+
+    async def get(
+        self, ten_env: AsyncTenEnv, text: str
+    ) -> AsyncIterator[bytes]:
+        payload = {
+            "api_key": self.config.api_key,
+            "model": self.config.model,
+            "text": text,
+            "voice_id": self.config.voice_id,
+            "language": self.config.language,
+            "sample_rate": self.config.sample_rate,
+        }
+
+        ten_env.log_info(f"payload: {payload}")
+
+        headers = {
+            "accept": "application/json",
+            # "Authorization": f"Bearer {self.config.api_key}",
+            "Content-Type": "application/json",
+        }
+
+        start_time = datetime.now()
+        ten_env.log_info(f"Start request, url: {self.config.url}, text: {text}")
+        # ttfb = None
+
+        async with aiohttp.ClientSession() as session:
+            try:
+                async with session.post(
+                    self.config.url,
+                    headers=headers,
+                    json=payload,
+                ) as response:
+                    if response.status != 200:
+                        raise RuntimeError(
+                            f"Request failed with status {response.status}"
+                        )
+
+                    first_chunk = True
+                    async for chunk in response.content.iter_any():
+                        ten_env.log_info("Received chunk of audio data.")
+                        if first_chunk:
+                            yield chunk[44:]  # 去除前44个byte
+                            first_chunk = False
+                        else:
+                            yield chunk
+            except aiohttp.ClientError as e:
+                ten_env.log_error(f"Client error occurred: {e}")
+            except asyncio.TimeoutError:
+                ten_env.log_error("Request timed out")
+            finally:
+                ten_env.log_info(
+                    f"http loop done, cost_time {self._duration_in_ms_since(start_time)}ms"
+                )
+
+        return
+
+    def _duration_in_ms(self, start: datetime, end: datetime) -> int:
+        return int((end - start).total_seconds() * 1000)
+
+    def _duration_in_ms_since(self, start: datetime) -> int:
+        return self._duration_in_ms(start, datetime.now())

ai_agents/agents/ten_packages/extension/cloudsway_tts_python/extension.py

@@ -0,0 +1,60 @@
+#
+# This file is part of TEN Framework, an open source project.
+# Licensed under the Apache License, Version 2.0.
+# See the LICENSE file for more information.
+#
+import traceback
+from ten_ai_base.transcription import AssistantTranscription
+from ten_ai_base.tts import AsyncTTSBaseExtension
+from .cloudsway_tts import CloudswayTTS, CloudswayTTSConfig
+from ten_runtime import (
+    AsyncTenEnv,
+)
+
+
+class CloudswayTTSExtension(AsyncTTSBaseExtension):
+    def __init__(self, name: str):
+        super().__init__(name)
+        self.client = None
+
+    async def on_init(self, ten_env: AsyncTenEnv) -> None:
+        await super().on_init(ten_env)
+        ten_env.log_debug("on_init")
+
+    async def on_start(self, ten_env: AsyncTenEnv) -> None:
+        await super().on_start(ten_env)
+        ten_env.log_debug("on_start")
+
+        config = await CloudswayTTSConfig.create_async(ten_env=ten_env)
+
+        ten_env.log_info(f"config: {config.api_key}")
+
+        if not config.api_key:
+            raise ValueError("api_key are required")
+
+        self.client = CloudswayTTS(config)
+
+    async def on_stop(self, ten_env: AsyncTenEnv) -> None:
+        await super().on_stop(ten_env)
+        ten_env.log_debug("on_stop")
+
+    async def on_deinit(self, ten_env: AsyncTenEnv) -> None:
+        await super().on_deinit(ten_env)
+        ten_env.log_debug("on_deinit")
+
+    async def on_request_tts(
+        self, ten_env: AsyncTenEnv, t: AssistantTranscription
+    ) -> None:
+        try:
+            data = self.client.get(ten_env, t.text)
+            async for frame in data:
+                await self.send_audio_out(
+                    ten_env, frame, sample_rate=self.client.config.sample_rate
+                )
+        except Exception:
+            ten_env.log_error(
+                f"on_request_tts failed: {traceback.format_exc()}"
+            )
+
+    async def on_cancel_tts(self, ten_env: AsyncTenEnv) -> None:
+        return await super().on_cancel_tts(ten_env)

ai_agents/agents/ten_packages/extension/cloudsway_tts_python/manifest.json

@@ -0,0 +1,74 @@
+{
+  "type": "extension",
+  "name": "cloudsway_tts_python",
+  "version": "0.1.0",
+  "dependencies": [
+    {
+      "type": "system",
+      "name": "ten_runtime_python",
+      "version": "0.8"
+    }
+  ],
+  "package": {
+    "include": [
+      "manifest.json",
+      "property.json",
+      "BUILD.gn",
+      "**.tent",
+      "**.py",
+      "README.md",
+      "tests/**"
+    ]
+  },
+  "api": {
+    "property": {
+      "properties": {
+        "api_key": {
+          "type": "string"
+        },
+        "model": {
+          "type": "string"
+        },
+        "sample_rate": {
+          "type": "int64"
+        },
+        "url": {
+          "type": "string"
+        },
+        "voice_id": {
+          "type": "string"
+        },
+        "language": {
+          "type": "string"
+        }
+      }
+    },
+    "data_in": [
+      {
+        "name": "text_data",
+        "property": {
+          "properties": {
+            "text": {
+              "type": "string"
+            }
+          }
+        }
+      }
+    ],
+    "cmd_in": [
+      {
+        "name": "flush"
+      }
+    ],
+    "cmd_out": [
+      {
+        "name": "flush"
+      }
+    ],
+    "audio_frame_out": [
+      {
+        "name": "pcm_frame"
+      }
+    ]
+  }
+}

ai_agents/agents/ten_packages/extension/cloudsway_tts_python/property.json

@@ -0,0 +1,8 @@
+{
+    "api_key": "",
+    "model": "cloudsway_tts",
+    "sample_rate": 32000,
+    "url": "http://0.0.0.0:9880/tts/stream",
+    "voice_id": "Luna_moan_1",
+    "language": "en"
+}

ai_agents/agents/ten_packages/extension/cloudsway_tts_python/requirements.txt

@@ -0,0 +1 @@
+aiohttp

ai_agents/docker-compose.yml

@@ -19,39 +19,33 @@ services:
     working_dir: /app
     env_file:
       - .env
-    networks:
-      - ten_agent_network
+    network_mode: host
   ten_agent_playground:
     image: ghcr.io/ten-framework/ten_agent_playground:0.10.31-25-g85c777953
     container_name: ten_agent_playground
     restart: always
     ports:
       - "3000:3000"
-    networks:
-      - ten_agent_network
+    network_mode: host
     environment:
-      - AGENT_SERVER_URL=http://ten_agent_dev:8080
-      - TEN_DEV_SERVER_URL=http://ten_agent_dev:49483
+      - AGENT_SERVER_URL=http://localhost:8080
+      - TEN_DEV_SERVER_URL=http://localhost:49483
   ten_agent_demo:
     image: ghcr.io/ten-framework/ten_agent_demo:0.10.36-7-gbd11efef5
     container_name: ten_agent_demo
     restart: always
     ports:
       - "3002:3000"
-    networks:
-      - ten_agent_network
+    network_mode: host
     environment:
-      - AGENT_SERVER_URL=http://ten_agent_dev:8080
+      - AGENT_SERVER_URL=http://localhost:8080
   # ten_graph_designer:
   #   image: ghcr.io/ten-framework/ten_graph_designer:4cc33b8
   #   container_name: ten_graph_designer
   #   restart: always
   #   ports:
   #     - "3001:3000"
   #   networks:
-  #     - ten_agent_network
+  #     - host
   #   environment:
   #     - TEN_DEV_SERVER_URL=http://ten_agent_dev:49483
-networks:
-  ten_agent_network:
-    driver: bridge

ai_agents/playground/src/common/moduleConfig.ts

@@ -135,6 +135,11 @@ export const ttsModuleRegistry: Record<string, ModuleRegistry.Module> = {
         type: ModuleRegistry.ModuleType.TTS,
         label: "Minimax TTS",
     },
+    cloudsway_tts_python: {
+        name: "cloudsway_tts_python",
+        type: ModuleRegistry.ModuleType.TTS,
+        label: "Cloudsway TTS",
+    },
     polly_tts: {
         name: "polly_tts",
         type: ModuleRegistry.ModuleType.TTS,