|
4 | 4 | * SPDX-License-Identifier: BSD 2-Clause License |
5 | 5 | */ |
6 | 6 |
|
7 | | -import Daily from "@daily-co/daily-js"; |
| 7 | +import { PipecatClient } from "@pipecat-ai/client-js"; |
| 8 | +import { DailyTransport } from "@pipecat-ai/daily-transport"; |
8 | 9 |
|
9 | 10 | /** |
10 | 11 | * ChatbotClient handles the connection and media management for a real-time |
11 | | - * voice interaction with an AI bot. |
| 12 | + * voice interaction with an AI bot using the Pipecat client SDK. |
| 13 | + * |
| 14 | + * The bot-ready handshake is handled by RTVI: the SDK signals client-ready |
| 15 | + * automatically once the transport reaches the `ready` state, the bot replies |
| 16 | + * with `bot-ready`, and only then does the bot push its first TTS frame. No |
| 17 | + * manual sendAppMessage / "playable" plumbing is needed. |
12 | 18 | */ |
13 | 19 | class ChatbotClient { |
14 | 20 | constructor() { |
15 | | - // Initialize client state |
16 | | - this.dailyCallObject = null; |
| 21 | + this.pcClient = null; |
| 22 | + this.botAudio = null; |
17 | 23 | this.setupDOMElements(); |
18 | 24 | this.setupEventListeners(); |
19 | 25 | } |
20 | 26 |
|
21 | | - /** |
22 | | - * Set up references to DOM elements and create necessary media elements |
23 | | - */ |
24 | 27 | setupDOMElements() { |
25 | | - // Get references to UI control elements |
26 | 28 | this.connectBtn = document.getElementById('connect-btn'); |
27 | 29 | this.disconnectBtn = document.getElementById('disconnect-btn'); |
28 | 30 | this.statusSpan = document.getElementById('connection-status'); |
29 | 31 | this.debugLog = document.getElementById('debug-log'); |
30 | | - |
31 | | - // Create an audio element for bot's voice output |
32 | | - this.botAudio = document.createElement('audio'); |
33 | | - this.botAudio.autoplay = true; |
34 | | - this.botAudio.playsInline = true; |
35 | | - document.body.appendChild(this.botAudio); |
36 | 32 | } |
37 | 33 |
|
38 | | - /** |
39 | | - * Set up event listeners for connect/disconnect buttons |
40 | | - */ |
41 | 34 | setupEventListeners() { |
42 | 35 | this.connectBtn.addEventListener('click', () => this.connect()); |
43 | 36 | this.disconnectBtn.addEventListener('click', () => this.disconnect()); |
44 | 37 | } |
45 | 38 |
|
46 | | - /** |
47 | | - * Add a timestamped message to the debug log |
48 | | - */ |
49 | 39 | log(message) { |
50 | 40 | const entry = document.createElement('div'); |
51 | 41 | entry.textContent = `${new Date().toISOString()} - ${message}`; |
52 | 42 |
|
53 | | - // Add styling based on message type |
54 | 43 | if (message.startsWith('User: ')) { |
55 | | - entry.style.color = '#2196F3'; // blue for user |
| 44 | + entry.style.color = '#2196F3'; |
56 | 45 | } else if (message.startsWith('Bot: ')) { |
57 | | - entry.style.color = '#4CAF50'; // green for bot |
| 46 | + entry.style.color = '#4CAF50'; |
58 | 47 | } |
59 | 48 |
|
60 | 49 | this.debugLog.appendChild(entry); |
61 | 50 | this.debugLog.scrollTop = this.debugLog.scrollHeight; |
62 | 51 | console.log(message); |
63 | 52 | } |
64 | 53 |
|
65 | | - /** |
66 | | - * Update the connection status display |
67 | | - */ |
68 | 54 | updateStatus(status) { |
69 | 55 | this.statusSpan.textContent = status; |
70 | 56 | this.log(`Status: ${status}`); |
71 | 57 | } |
72 | 58 |
|
73 | | - handleEventToConsole (evt) { |
74 | | - this.log(`Received event: ${evt.action}`); |
75 | | - }; |
76 | | - |
77 | 59 | /** |
78 | | - * Set up listeners for track events (start/stop) |
79 | | - * This handles new tracks being added during the session |
| 60 | + * Attach the bot's audio track to a hidden <audio> element so it plays back. |
| 61 | + * The Pipecat client SDK fires onTrackStarted for every remote track. |
80 | 62 | */ |
81 | | - setupTrackListeners() { |
82 | | - if (!this.dailyCallObject) return; |
83 | | - |
84 | | - this.dailyCallObject.on("joined-meeting", () => { |
85 | | - this.updateStatus('Connected'); |
86 | | - this.connectBtn.disabled = true; |
87 | | - this.disconnectBtn.disabled = false; |
88 | | - this.log('Client connected'); |
89 | | - }); |
90 | | - this.dailyCallObject.on("track-started", (evt) => { |
91 | | - if (evt.track.kind === "audio" && evt.participant.local === false) { |
92 | | - this.log("Audio track started.") |
93 | | - this.setupAudioTrack(evt.track); |
94 | | - } |
95 | | - }); |
96 | | - this.dailyCallObject.on("track-stopped", this.handleEventToConsole.bind(this)); |
97 | | - this.dailyCallObject.on("participant-joined", this.handleEventToConsole.bind(this)); |
98 | | - this.dailyCallObject.on("participant-updated", this.handleEventToConsole.bind(this)); |
99 | | - this.dailyCallObject.on("participant-left", () => { |
100 | | - // When the bot leaves, we are also disconnecting from the call |
101 | | - this.disconnect() |
102 | | - }); |
103 | | - this.dailyCallObject.on("left-meeting", () => { |
104 | | - this.updateStatus('Disconnected'); |
105 | | - this.connectBtn.disabled = false; |
106 | | - this.disconnectBtn.disabled = true; |
107 | | - this.log('Client disconnected'); |
108 | | - }); |
109 | | - this.dailyCallObject.on("error", this.handleEventToConsole.bind(this)); |
110 | | - } |
| 63 | + handleBotAudio(track, participant) { |
| 64 | + if (participant?.local || track.kind !== 'audio') return; |
111 | 65 |
|
112 | | - /** |
113 | | - * Set up an audio track for playback |
114 | | - * Handles both initial setup and track updates |
115 | | - */ |
116 | | - setupAudioTrack(track) { |
117 | | - this.log(`Setting up audio track, track state: ${track.readyState}, muted: ${track.muted}`); |
| 66 | + this.log('Bot audio track started.'); |
118 | 67 |
|
119 | | - // Check if we're already playing this track |
120 | | - if (this.botAudio.srcObject) { |
121 | | - const oldTrack = this.botAudio.srcObject.getAudioTracks()[0]; |
122 | | - if (oldTrack?.id === track.id) return; |
| 68 | + if (!this.botAudio) { |
| 69 | + this.botAudio = document.createElement('audio'); |
| 70 | + this.botAudio.autoplay = true; |
| 71 | + this.botAudio.playsInline = true; |
| 72 | + document.body.appendChild(this.botAudio); |
123 | 73 | } |
124 | | - // Create a new MediaStream with the track and set it as the audio source |
| 74 | + |
125 | 75 | this.botAudio.srcObject = new MediaStream([track]); |
126 | | - this.botAudio.onplaying = async (event) => { |
127 | | - this.log("onplaying") |
128 | | - this.log("Will send the audio message to play the audio at the next tick") |
129 | | - this.dailyCallObject.sendAppMessage("playable") |
130 | | - } |
131 | 76 | } |
132 | 77 |
|
133 | | - async fetchRoomInfo() { |
134 | | - let connectUrl = '/connect' |
135 | | - let res = await fetch(connectUrl, { |
136 | | - method: "POST", |
137 | | - mode: "cors", |
138 | | - headers: new Headers({ |
139 | | - "Content-Type": "application/json" |
140 | | - }), |
141 | | - }) |
142 | | - if (res.ok) { |
143 | | - return res.json(); |
| 78 | + removeBotAudio() { |
| 79 | + if (this.botAudio) { |
| 80 | + const stream = this.botAudio.srcObject; |
| 81 | + if (stream) { |
| 82 | + stream.getTracks().forEach((t) => t.stop()); |
| 83 | + } |
| 84 | + this.botAudio.srcObject = null; |
| 85 | + this.botAudio.remove(); |
| 86 | + this.botAudio = null; |
144 | 87 | } |
145 | 88 | } |
146 | 89 |
|
147 | | - /** |
148 | | - * Initialize and connect to the bot |
149 | | - * This sets up the RTVI client, initializes devices, and establishes the connection |
150 | | - */ |
151 | 90 | async connect() { |
152 | 91 | try { |
153 | | - // Initialize the client |
154 | | - this.dailyCallObject = Daily.createCallObject({ |
155 | | - subscribeToTracksAutomatically: true, |
| 92 | + this.pcClient = new PipecatClient({ |
| 93 | + transport: new DailyTransport(), |
| 94 | + enableMic: true, |
| 95 | + enableCam: false, |
| 96 | + callbacks: { |
| 97 | + onTransportStateChanged: (state) => { |
| 98 | + this.updateStatus(state); |
| 99 | + const isConnected = state === 'ready'; |
| 100 | + this.connectBtn.disabled = state !== 'idle' && state !== 'disconnected'; |
| 101 | + this.disconnectBtn.disabled = !isConnected; |
| 102 | + }, |
| 103 | + onBotReady: () => { |
| 104 | + this.log('Bot ready: greeting will play next.'); |
| 105 | + }, |
| 106 | + onTrackStarted: (track, participant) => this.handleBotAudio(track, participant), |
| 107 | + onDisconnected: () => { |
| 108 | + this.log('Disconnected from bot.'); |
| 109 | + this.removeBotAudio(); |
| 110 | + this.connectBtn.disabled = false; |
| 111 | + this.disconnectBtn.disabled = true; |
| 112 | + }, |
| 113 | + onError: (message) => { |
| 114 | + this.log(`Error: ${message?.data?.message || message}`); |
| 115 | + }, |
| 116 | + }, |
156 | 117 | }); |
157 | 118 |
|
158 | | - // Set up listeners for media track events |
159 | | - this.setupTrackListeners(); |
| 119 | + // Expose for debugging. |
| 120 | + window.pcClient = this.pcClient; |
160 | 121 |
|
161 | 122 | this.log('Creating the bot...'); |
162 | | - let roomInfo = await this.fetchRoomInfo() |
163 | | - |
164 | | - // Connect to the bot |
165 | | - this.log('Connecting to bot...'); |
166 | | - // Only for making debugger easier |
167 | | - window.callObject = this.dailyCallObject; |
168 | | - await this.dailyCallObject.join({ |
169 | | - url: roomInfo.room_url, |
170 | | - }); |
171 | | - |
172 | | - this.log('Connection complete'); |
| 123 | + await this.pcClient.startBotAndConnect({ endpoint: '/connect' }); |
| 124 | + this.log('Connection complete.'); |
173 | 125 | } catch (error) { |
174 | | - // Handle any errors during connection |
175 | 126 | this.log(`Error connecting: ${error.message}`); |
176 | 127 | this.log(`Error stack: ${error.stack}`); |
177 | 128 | this.updateStatus('Error'); |
178 | 129 |
|
179 | | - // Clean up if there's an error |
180 | | - if (this.dailyCallObject) { |
| 130 | + if (this.pcClient) { |
181 | 131 | try { |
182 | | - await this.dailyCallObject.leave(); |
| 132 | + await this.pcClient.disconnect(); |
183 | 133 | } catch (disconnectError) { |
184 | 134 | this.log(`Error during disconnect: ${disconnectError.message}`); |
185 | 135 | } |
186 | 136 | } |
187 | 137 | } |
188 | 138 | } |
189 | 139 |
|
190 | | - /** |
191 | | - * Disconnect from the bot and clean up media resources |
192 | | - */ |
193 | 140 | async disconnect() { |
194 | | - if (this.dailyCallObject) { |
| 141 | + if (this.pcClient) { |
195 | 142 | try { |
196 | | - // Disconnect the RTVI client |
197 | | - await this.dailyCallObject.leave(); |
198 | | - await this.dailyCallObject.destroy(); |
199 | | - this.dailyCallObject = null; |
200 | | - |
201 | | - // Clean up audio |
202 | | - if (this.botAudio.srcObject) { |
203 | | - this.botAudio.srcObject.getTracks().forEach((track) => track.stop()); |
204 | | - this.botAudio.srcObject = null; |
205 | | - } |
| 143 | + await this.pcClient.disconnect(); |
206 | 144 | } catch (error) { |
207 | 145 | this.log(`Error disconnecting: ${error.message}`); |
| 146 | + } finally { |
| 147 | + this.pcClient = null; |
| 148 | + this.removeBotAudio(); |
208 | 149 | } |
209 | 150 | } |
210 | 151 | } |
211 | 152 | } |
212 | 153 |
|
213 | | -// Initialize the client when the page loads |
214 | 154 | window.addEventListener('DOMContentLoaded', () => { |
215 | 155 | new ChatbotClient(); |
216 | 156 | }); |
0 commit comments