From 2d01db74daa931b87475ee34c48d63cd68b2857a Mon Sep 17 00:00:00 2001 From: Long Chen Date: Wed, 19 Feb 2025 00:08:27 +0800 Subject: [PATCH 1/3] add datastream text input --- package-lock.json | 101 ++++++++++++----------- package.json | 8 +- src/hooks/useSendText.tsx | 22 +++++ src/transcriptions/TranscriptionTile.tsx | 12 ++- 4 files changed, 91 insertions(+), 52 deletions(-) create mode 100644 src/hooks/useSendText.tsx diff --git a/package-lock.json b/package-lock.json index ebb468eb..941bb1f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,14 +8,14 @@ "name": "agents-playground", "version": "0.1.0", "dependencies": { - "@livekit/components-react": "^2.6.0", - "@livekit/components-styles": "^1.1.1", + "@livekit/components-react": "^2.8.1", + "@livekit/components-styles": "^1.1.4", "@radix-ui/react-dropdown-menu": "^2.0.6", "cookies-next": "^4.1.1", "framer-motion": "^10.16.16", "js-yaml": "^4.1.0", - "livekit-client": "^2.5.1", - "livekit-server-sdk": "^2.6.1", + "livekit-client": "^2.9.2", + "livekit-server-sdk": "^2.10.1", "lodash": "^4.17.21", "next": "^14.0.4", "next-plugin-preval": "^1.2.6", @@ -2030,9 +2030,9 @@ } }, "node_modules/@livekit/components-core": { - "version": "0.11.11", - "resolved": "https://registry.npmjs.org/@livekit/components-core/-/components-core-0.11.11.tgz", - "integrity": "sha512-r6VgeS7zq9lMKiNJHSKUHMbmuirksEvHXEGsUvecsHTkZD9nhbhCB/01qCVaCVHVnLXvAYycNXVD4qcepPOpFQ==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@livekit/components-core/-/components-core-0.12.1.tgz", + "integrity": "sha512-R7qWoVzPckOYxEHZgP3Kp8u+amu+isnTptgoZV7+bpmLRBHI7mWnaD+0uDWlyIMjI1pBbK3wHg0ILKa5UytI+A==", "license": "Apache-2.0", "dependencies": { "@floating-ui/dom": "1.6.11", @@ -2043,17 +2043,17 @@ "node": ">=18" }, "peerDependencies": { - "livekit-client": "^2.5.7", + "livekit-client": "^2.8.1", "tslib": "^2.6.2" } }, "node_modules/@livekit/components-react": { - "version": "2.6.11", - "resolved": "https://registry.npmjs.org/@livekit/components-react/-/components-react-2.6.11.tgz", - "integrity": "sha512-33V5D2yyt4p9qqjHSMFeu0mUl0S0Qd3zl+x4in9jNl3ZMZBwds9aFx+BtgAGymlAbjbxWPWmHn/SDI3R5/hJRQ==", + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/@livekit/components-react/-/components-react-2.8.1.tgz", + "integrity": "sha512-XpuDu7iDMcN4pkV8CYNzHf9hLNdYOeEtbmCr7Zesy6Au3BxUl4aS1Ajmg0b75Rx7zTlkyCJt9Lm4VrEqbJCI6Q==", "license": "Apache-2.0", "dependencies": { - "@livekit/components-core": "0.11.11", + "@livekit/components-core": "0.12.1", "clsx": "2.1.1", "usehooks-ts": "3.1.0" }, @@ -2062,7 +2062,7 @@ }, "peerDependencies": { "@livekit/krisp-noise-filter": "^0.2.12", - "livekit-client": "^2.5.7", + "livekit-client": "^2.8.1", "react": ">=18", "react-dom": ">=18", "tslib": "^2.6.2" @@ -2083,15 +2083,15 @@ } }, "node_modules/@livekit/mutex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@livekit/mutex/-/mutex-1.0.0.tgz", - "integrity": "sha512-aiUhoThBNF9UyGTxEURFzJLhhPLIVTnQiEVMjRhPnfHNKLfo2JY9xovHKIus7B78UD5hsP6DlgpmAsjrz4U0Iw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@livekit/mutex/-/mutex-1.1.1.tgz", + "integrity": "sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==", "license": "Apache-2.0" }, "node_modules/@livekit/protocol": { - "version": "1.29.4", - "resolved": "https://registry.npmjs.org/@livekit/protocol/-/protocol-1.29.4.tgz", - "integrity": "sha512-dsqxvABHilrMA0BU5m1w8cMWSVeDjV2ZUIUDClNQZju3c30DLMfEYDHU5nmXDfaaHjNIgoRbYR7upJMozG8JJg==", + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/@livekit/protocol/-/protocol-1.33.0.tgz", + "integrity": "sha512-361mBlFgI3nvn8oSQIL38gDUBGbOSwsEOqPgX0c1Jwz75/sD/TTvPeAM4zAz6OrV5Q4vI4Ruswecnyv5SG4oig==", "license": "Apache-2.0", "dependencies": { "@bufbuild/protobuf": "^1.10.0" @@ -5992,29 +5992,43 @@ "dev": true }, "node_modules/livekit-client": { - "version": "2.7.5", - "resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-2.7.5.tgz", - "integrity": "sha512-sPhHYwXvG75y1LDC50dDC9k6Z49L2vc/HcMRhzhi7yBca6ofPEebpB0bmPOry4ovrnFA+a8TL1pFR2mko1/clw==", + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-2.9.2.tgz", + "integrity": "sha512-CEURRYoVvJGBAlY0UJAGx5mpC/jE90rw2mor6eYYyu2LLNflh03Rfy+mV5o2CS1xNJD7LS+PTH0sk0ZvxXC22g==", "license": "Apache-2.0", "dependencies": { - "@livekit/mutex": "1.0.0", - "@livekit/protocol": "1.29.4", + "@livekit/mutex": "1.1.1", + "@livekit/protocol": "1.33.0", "events": "^3.3.0", - "loglevel": "^1.8.0", - "sdp-transform": "^2.14.1", + "loglevel": "^1.9.2", + "sdp-transform": "^2.15.0", "ts-debounce": "^4.0.0", - "tslib": "2.7.0", + "tslib": "2.8.1", "typed-emitter": "^2.1.0", - "webrtc-adapter": "^9.0.0" + "webrtc-adapter": "^9.0.1" + } + }, + "node_modules/livekit-client/node_modules/loglevel": { + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.9.2.tgz", + "integrity": "sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==", + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + }, + "funding": { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/loglevel" } }, "node_modules/livekit-server-sdk": { - "version": "2.9.5", - "resolved": "https://registry.npmjs.org/livekit-server-sdk/-/livekit-server-sdk-2.9.5.tgz", - "integrity": "sha512-vx93zG9r0EdAbeK3/+XuoO6w2lHeOOo1RRgL49ZGT2gihLtWQlYP6YFcVNr2ere9NtYT0mfLoGQtDcktXKdYOQ==", + "version": "2.10.1", + "resolved": "https://registry.npmjs.org/livekit-server-sdk/-/livekit-server-sdk-2.10.1.tgz", + "integrity": "sha512-rzWCZzMFWa+Fl3Vp4kQwoOLcbvMvQEqe4WXl3XDp4u7R/Rg7TY3MZ4JcHv5mvIawqAJ81hR6x23C7+QWteI/dQ==", "license": "Apache-2.0", "dependencies": { - "@livekit/protocol": "^1.30.0", + "@bufbuild/protobuf": "^1.7.2", + "@livekit/protocol": "^1.32.1", "camelcase-keys": "^9.0.0", "jose": "^5.1.2" }, @@ -6022,15 +6036,6 @@ "node": ">=18" } }, - "node_modules/livekit-server-sdk/node_modules/@livekit/protocol": { - "version": "1.30.0", - "resolved": "https://registry.npmjs.org/@livekit/protocol/-/protocol-1.30.0.tgz", - "integrity": "sha512-SDI9ShVKj8N3oOSinr8inaxD3FXgmgoJlqN35uU/Yx1sdoDeQbzAuBFox7bYjM+VhnZ1V22ivIDjAsKr00H+XQ==", - "license": "Apache-2.0", - "dependencies": { - "@bufbuild/protobuf": "^1.10.0" - } - }, "node_modules/loader-runner": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz", @@ -6098,6 +6103,7 @@ "version": "1.9.1", "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.9.1.tgz", "integrity": "sha512-hP3I3kCrDIMuRwAwHltphhDM1r8i55H33GgqjXbrisuJhF4kRhW1dNuxsRklp4bXl8DSdLaNLuiL4A/LWRfxvg==", + "license": "MIT", "engines": { "node": ">= 0.6.0" }, @@ -7480,9 +7486,10 @@ "integrity": "sha512-d7wDPgDV3DDiqulJjKiV2865wKsJ34YI+NDREbm+FySq6WuKOikwyNQcm+doLAZ1O6ltdO0SeKle2xMpN3Brgw==" }, "node_modules/sdp-transform": { - "version": "2.14.2", - "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.14.2.tgz", - "integrity": "sha512-icY6jVao7MfKCieyo1AyxFYm1baiM+fA00qW/KrNNVlkxHAd34riEKuEkUe4bBb3gJwLJZM+xT60Yj1QL8rHiA==", + "version": "2.15.0", + "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.15.0.tgz", + "integrity": "sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw==", + "license": "MIT", "bin": { "sdp-verify": "checker.js" } @@ -8112,9 +8119,9 @@ } }, "node_modules/tslib": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", - "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==", + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, "node_modules/type-check": { diff --git a/package.json b/package.json index b3fbc7a2..cad49e39 100644 --- a/package.json +++ b/package.json @@ -9,14 +9,14 @@ "lint": "next lint" }, "dependencies": { - "@livekit/components-react": "^2.6.0", - "@livekit/components-styles": "^1.1.1", + "@livekit/components-react": "^2.8.1", + "@livekit/components-styles": "^1.1.4", "@radix-ui/react-dropdown-menu": "^2.0.6", "cookies-next": "^4.1.1", "framer-motion": "^10.16.16", "js-yaml": "^4.1.0", - "livekit-client": "^2.5.1", - "livekit-server-sdk": "^2.6.1", + "livekit-client": "^2.9.2", + "livekit-server-sdk": "^2.10.1", "lodash": "^4.17.21", "next": "^14.0.4", "next-plugin-preval": "^1.2.6", diff --git a/src/hooks/useSendText.tsx b/src/hooks/useSendText.tsx new file mode 100644 index 00000000..fd1b4d3f --- /dev/null +++ b/src/hooks/useSendText.tsx @@ -0,0 +1,22 @@ +import { useCallback } from 'react'; +import { useRoomContext } from '@livekit/components-react'; + +export const useSendText = () => { + const room = useRoomContext(); + + const sendText = useCallback(async (message: string) => { + if (!room?.localParticipant) return; + + try { + console.log('Sending text message:', message); + const info = await room.localParticipant.sendText(message, { + topic: 'lk.room_text_input' + }); + console.log('Text message sent:', info); + } catch (e) { + console.error('Failed to send message:', e); + } + }, [room]); + + return sendText; +}; \ No newline at end of file diff --git a/src/transcriptions/TranscriptionTile.tsx b/src/transcriptions/TranscriptionTile.tsx index b168a850..6a7aa2df 100644 --- a/src/transcriptions/TranscriptionTile.tsx +++ b/src/transcriptions/TranscriptionTile.tsx @@ -12,6 +12,7 @@ import { TranscriptionSegment, } from "livekit-client"; import { useEffect, useState } from "react"; +import { useSendText } from '@/hooks/useSendText'; export function TranscriptionTile({ agentAudioTrack, @@ -33,6 +34,7 @@ export function TranscriptionTile({ ); const [messages, setMessages] = useState([]); const { chatMessages, send: sendChat } = useChat(); + const sendText = useSendText(); // store transcripts useEffect(() => { @@ -91,8 +93,16 @@ export function TranscriptionTile({ localMessages.segments, ]); + const handleSendMessage = async (message: string) => { + const [chatMessage, _] = await Promise.all([ + sendChat(message), + sendText(message) + ]); + return chatMessage; + }; + return ( - + ); } From a6798720fd919a9887e05f6e3f2662b6862ba8ae Mon Sep 17 00:00:00 2001 From: Long Chen Date: Wed, 19 Feb 2025 20:29:23 +0800 Subject: [PATCH 2/3] upgrade deps --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 941bb1f4..1ceb2ada 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "cookies-next": "^4.1.1", "framer-motion": "^10.16.16", "js-yaml": "^4.1.0", - "livekit-client": "^2.9.2", + "livekit-client": "^2.9.3", "livekit-server-sdk": "^2.10.1", "lodash": "^4.17.21", "next": "^14.0.4", @@ -5992,9 +5992,9 @@ "dev": true }, "node_modules/livekit-client": { - "version": "2.9.2", - "resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-2.9.2.tgz", - "integrity": "sha512-CEURRYoVvJGBAlY0UJAGx5mpC/jE90rw2mor6eYYyu2LLNflh03Rfy+mV5o2CS1xNJD7LS+PTH0sk0ZvxXC22g==", + "version": "2.9.3", + "resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-2.9.3.tgz", + "integrity": "sha512-mDLgenjmGI3ga1qXsuonw8cTpQMUTm6T8tmjB/O+uzjsjI9UP0ebzjgrcOBn+IamjbulIdWWvtrt5A/6ftuH5A==", "license": "Apache-2.0", "dependencies": { "@livekit/mutex": "1.1.1", diff --git a/package.json b/package.json index cad49e39..560aba9d 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "cookies-next": "^4.1.1", "framer-motion": "^10.16.16", "js-yaml": "^4.1.0", - "livekit-client": "^2.9.2", + "livekit-client": "^2.9.3", "livekit-server-sdk": "^2.10.1", "lodash": "^4.17.21", "next": "^14.0.4", From d1a9bf57f95d32e164e9afb9d737d6d3403e43e3 Mon Sep 17 00:00:00 2001 From: Long Chen Date: Mon, 24 Feb 2025 11:24:49 +0800 Subject: [PATCH 3/3] fix rpc call with publish_for --- src/components/playground/Playground.tsx | 6 ++++-- src/hooks/useSendText.tsx | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/components/playground/Playground.tsx b/src/components/playground/Playground.tsx index 8986fa1b..9084b9f0 100644 --- a/src/components/playground/Playground.tsx +++ b/src/components/playground/Playground.tsx @@ -221,9 +221,11 @@ export default function Playground({ const handleRpcCall = useCallback(async () => { if (!voiceAssistant.agent || !room) return; + const agent_identity = voiceAssistant.agentAttributes?.['lk.publish_for'] || voiceAssistant.agent.identity; + console.log('Agent identity for RPC call:', agent_identity); try { const response = await room.localParticipant.performRpc({ - destinationIdentity: voiceAssistant.agent.identity, + destinationIdentity: agent_identity, method: rpcMethod, payload: rpcPayload, }); @@ -231,7 +233,7 @@ export default function Playground({ } catch (e) { console.error('RPC call failed:', e); } - }, [room, rpcMethod, rpcPayload, voiceAssistant.agent]); + }, [room, rpcMethod, rpcPayload, voiceAssistant.agent, voiceAssistant.agentAttributes]); const settingsTileContent = useMemo(() => { return ( diff --git a/src/hooks/useSendText.tsx b/src/hooks/useSendText.tsx index fd1b4d3f..36bcd2f1 100644 --- a/src/hooks/useSendText.tsx +++ b/src/hooks/useSendText.tsx @@ -10,7 +10,7 @@ export const useSendText = () => { try { console.log('Sending text message:', message); const info = await room.localParticipant.sendText(message, { - topic: 'lk.room_text_input' + topic: 'lk.chat' }); console.log('Text message sent:', info); } catch (e) {