let peerConnection; let stream; let webrtc_id; let isMuted = true; let pendingCandidates = []; let isMicOpen = false; const audioOutput = document.getElementById('audio-output'); const startButton = document.getElementById('start-button'); const talkButton = document.getElementById('hold-to-talk-button'); const speakerButton = document.getElementById('speaker-button') const selectAIVoice = document.getElementById('select-ai-voice'); const stopAnswerButton = document.getElementById('stop-answer-button'); const selectVoiceActivity = document.getElementById('select-restart-ai'); const restartConversationButton = document.getElementById('restart-convo-button'); const agentIDInput = document.getElementById('agent-id-input'); const agentIDButton = document.getElementById('agent-id-button'); const agentNameLabel = document.getElementById('agent-name'); const rtcid = document.getElementById('rtcid'); // mute audio output by default audioOutput.muted = isMuted; function updateButtonState() { const button = document.getElementById('start-button'); button.innerHTML = ''; if (peerConnection && (peerConnection.connectionState === 'connecting' || peerConnection.connectionState === 'new')) { const text = document.createElement('span'); text.textContent = 'Connecting...'; button.appendChild(text); button.disabled = true; } else if (peerConnection && peerConnection.connectionState === 'connected') { const text = document.createElement('span'); text.textContent = 'Disconnect'; button.appendChild(text); button.disabled = false; } else { const text = document.createElement('span'); text.textContent = 'Connect'; button.appendChild(text); button.disabled = false; } } function showError(message) { const toast = document.getElementById('toast-msg'); toast.textContent = message; toast.className = 'toast error'; toast.style.display = 'block'; // Hide toast after 5 seconds setTimeout(() => { toast.style.display = 'none'; }, 5000); } function showWarning(message) { const toast = document.getElementById('toast-msg'); toast.textContent = message; toast.className = 'toast warning'; toast.style.display = 'block'; // Hide toast after 5 seconds setTimeout(() => { toast.style.display = 'none'; }, 5000); } function showInfo(message) { const toast = document.getElementById('toast-msg'); toast.textContent = message; toast.className = 'toast'; toast.style.display = 'block'; // Hide toast after 5 seconds setTimeout(() => { toast.style.display = 'none'; }, 5000); } async function fetchUntilOpenAIConnected(url, maxRetries = 10, delayMs = 1000) { let attempts = 0; while (true) { try { const response = await fetch(url); const data = await response.json(); if (!data.status) throw new Error(`OpenAI Error! Connected: ${data.status}`); // Success return data; } catch (error) { attempts++; console.warn(`Attempt ${attempts} failed: ${error.message}`); if (attempts >= maxRetries) { throw new Error(`Max retries reached for ${url}`); } // ⏱ Wait before retrying (exponential backoff with jitter) const backoff = delayMs * Math.pow(2, attempts) + Math.random() * 500; await new Promise(resolve => setTimeout(resolve, backoff)); } } } function sendCandidate(candidate) { console.debug("Sending ICE candidate", candidate); console.debug("WebRTC ID: ", webrtc_id); fetch('/webrtc/offer', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ candidate: candidate.toJSON(), webrtc_id: webrtc_id, type: "ice-candidate", }) }); } async function setupWebRTC() { if (peerConnection) { stopWebRTC(); } isConnecting = true; const config = {"iceServers": [{"urls": "stun:stun.l.google.com:19302"}, {"urls": ["turn:34.65.7.112:443?transport=tcp", "turn:34.65.7.112:443", "turn:34.65.7.112:3478?transport=tcp", "turn:34.65.7.112:3478"], "username": "vidio", "credential": "7aa4ed8b41a23ea18d2bf856"}]}; peerConnection = new RTCPeerConnection(config); setElementDisabledState(true); updateButtonState(); const timeoutId = setTimeout(() => { showWarning("Connection is taking longer than usual. Are you on a VPN?"); }, 5000); try { stream = await navigator.mediaDevices.getUserMedia({ audio: true }); stream.getTracks().forEach(track => { peerConnection.addTrack(track, stream); }); peerConnection.addEventListener('track', (evt) => { if (audioOutput.srcObject !== evt.streams[0]) { audioOutput.srcObject = evt.streams[0]; audioOutput.play(); } }); peerConnection.onicecandidate = ({ candidate }) => { if (candidate) { if (peerConnection.remoteDescription) { sendCandidate(candidate); } else { pendingCandidates.push(candidate); } } }; const dataChannel = peerConnection.createDataChannel('text'); dataChannel.onmessage = (event) => { const eventJson = JSON.parse(event.data); if (eventJson.type === "error") { showError(eventJson.message); } else { console.debug('Received message:', eventJson); } }; const offer = await peerConnection.createOffer(); await peerConnection.setLocalDescription(offer); peerConnection.addEventListener('connectionstatechange', (event) => { console.debug('connectionstatechange', peerConnection.connectionState); if (peerConnection.connectionState === 'connected') { clearTimeout(timeoutId); const toast = document.getElementById('toast-msg'); toast.style.display = 'none'; } if (peerConnection.connectionState === 'failed' || peerConnection.connectionState === 'disconnected') { stopWebRTC() } }); webrtc_id = Math.random().toString(36).substring(7); console.log('Generated WebRTC ID: ', webrtc_id); const response = await fetch('/webrtc/offer', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ sdp: peerConnection.localDescription.sdp, type: peerConnection.localDescription.type, webrtc_id: webrtc_id }) }); const serverResponse = await response.json(); if (serverResponse.status === 'failed') { showError(serverResponse.meta.error === 'concurrency_limit_reached' ? `Too many connections. Maximum limit is ${serverResponse.meta.limit}` : serverResponse.meta.error); clearTimeout(timeoutId); stopWebRTC(); return; } await peerConnection.setRemoteDescription(serverResponse); if (pendingCandidates.length > 0) { console.debug('Sending Pending ICE Candidates'); pendingCandidates.forEach(c => sendCandidate(c)); pendingCandidates = []; } // Set up the RTC ID input field rtcid.value = webrtc_id; return true; } catch (err) { clearTimeout(timeoutId); console.error('Error setting up WebRTC:', err); showError('Failed to establish connection. Please try again.'); stopWebRTC(); return false; } } function stopWebRTC() { if (peerConnection) { if (peerConnection.getTransceivers) { peerConnection.getTransceivers().forEach(transceiver => { if (transceiver.stop) { transceiver.stop(); } }); } if (peerConnection.getSenders) { peerConnection.getSenders().forEach(sender => { if (sender.track && sender.track.stop) sender.track.stop(); }); } console.debug('closing'); peerConnection.close(); peerConnection = undefined webrtc_id = undefined rtcid.value = ''; } if (stream) { if (stream.getTrakcs) { stream.getTracks().forEach(track => { if (track.stop) track.stop(); }) } stream = undefined; } pendingCandidates = []; updateButtonState(); setElementDisabledState(true); } async function isOpenAIConnected() { try { console.debug("Checking OpenAI connection...") const data = await fetchUntilOpenAIConnected('/control/health/openai'); if (!data.status) { showWarning("OpenAI not connected, please press 'Connect' or wait!") } else { console.debug(`OpenAI connection: ${data.status}`) } return data.status; } catch (error) { console.error('OpenAI Health Check Error:', error); showError("OpenAI Health Check Failed!") return false; } } function toggleConnection() { console.debug(peerConnection, peerConnection?.connectionState); if (!peerConnection || (peerConnection.connectionState !== 'connecting' && peerConnection.connectionState !== 'connected')) { console.debug('starting webrtc'); connect(); } else { console.debug('stopping webrtc'); stopWebRTC(); } } function setElementDisabledState(toDisable) { //if peerConnection is undefined, then webrtc was disconnected so disable fields const disabled = toDisable || peerConnection == undefined; //set the disabled state for buttons that interact with OpenAI restartConversationButton.disabled = disabled; selectVoiceActivity.disabled = disabled; selectAIVoice.disabled = disabled; agentIDButton.disabled = disabled; agentIDInput.disabled = disabled; talkButton.disabled = disabled; stopAnswerButton.disabled = disabled; if(disabled) { agentNameLabel.classList.add('disabled'); } else { agentNameLabel.classList.remove('disabled'); } // can disable the button only if peerConnection is set startButton.disabled = toDisable && peerConnection; } async function toggleMicrophone(enable) { // isMicOpen !== enable -> small guard to avoid spamming enable=false with the mouseleave event if (isMicOpen !== enable && await isOpenAIConnected()) { isMicOpen = enable; await fetch(`/control/microphone?enable=${enable}`) .then(response => response.json()) .then(data => console.debug(`Response for toggle microphone (${enable}):`, data)) .catch(error => { console.error("Error:", error); showError(`Failed to switch microphone to '${enable}'`) }); } } function createVoiceOptGroups(data) { const supportedVoices = data.supported_voices; for (const [label, voices] of Object.entries(supportedVoices)) { const optGroup = document.createElement("optgroup"); optGroup.label = label.charAt(0).toUpperCase() + label.slice(1); voices.forEach(voice => { const opt = document.createElement("option"); opt.value = voice; opt.textContent = voice.charAt(0).toUpperCase() + voice.slice(1); optGroup.appendChild(opt); }); selectAIVoice.appendChild(optGroup); } } async function populateAIVoiceDropdown() { // Add default placeholder option const placeholderOption = document.createElement("option"); placeholderOption.value = ""; placeholderOption.textContent = "Select Voice"; placeholderOption.disabled = true; placeholderOption.hidden = true; selectAIVoice.innerHTML = ""; // Reset options selectAIVoice.appendChild(placeholderOption); try { const response = await fetch("/control/voice"); const data = await response.json(); createVoiceOptGroups(data); } catch (error) { console.error("Error fetching data:", error); showError("Unable to load voice list!") } selectAIVoice.selectedIndex = 0; } async function updateSelectedAIVoice() { const selectedVoice = selectAIVoice.value; selectAIVoice.selectedIndex = 0; if (selectedVoice && (await isOpenAIConnected())) { try { setElementDisabledState(true); const response = await fetch("/control/voice", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ voice: selectedVoice }) }); const result = await response.json(); console.debug("Voice Update:", result); setTimeout(() => { // Re-enable after 3 seconds setElementDisabledState(false); }, 3000); // 3000 milliseconds = 3 seconds } catch (error) { console.error("Error sending selected value:", error); showError("Failed to update voice!") // Re-enable on error setElementDisabledState(false); } } } // mute web playback function toggleSpeaker() { isMuted = !isMuted; audioOutput.muted = isMuted const button = document.getElementById("speaker-button"); // Toggle Class for Changing Icon if (isMuted) { button.classList.add("muted"); } else { button.classList.remove("muted"); } } async function stopAnswer() { if (await isOpenAIConnected()) { await fetch(`/control/conversation?action=stop`) .then(response => response.json()) .then(data => console.debug(`Response for stop answer:`, data)) .catch(error => { console.error("Error:", error); showError("Failed to stop the answer playback!") }); } } async function updateVAD() { const turnDetection = selectVoiceActivity.value; selectVoiceActivity.selectedIndex = 0; await restartConversation(turnDetection); } async function restartConversation(turnDetection, agentId) { const turnDetectionParam = turnDetection === undefined ? "" : `&turn_detection=${turnDetection}` const agentIdParam = agentId === undefined ? "" : `&agent_id=${agentId}` if (await isOpenAIConnected()) { console.debug(`Restarting Conversation: Turn Detection: ${turnDetection}, AgentID: ${agentId}`); setElementDisabledState(true); const url = `/control/conversation?action=restart${turnDetectionParam}${agentIdParam}` await fetch(url) .then(response => response.json()) .then(data => { console.debug(`Response for restart conversation:`, data) if (data.hasOwnProperty('message') && data.message != null) { const show = data.status ? showInfo : showError; show(data.message); } agentNameLabel.textContent = data?.agent?.name ?? 'default_agent'; agentIDInput.value = data?.agent?.id ?? '' setElementDisabledState(false); }) .catch(error => { console.error("Error:", error); // Re-enable on error setElementDisabledState(false); showError("Failed to restart conversation!") }); } } function configureVADDropdown() { // set the selected index to the hidden 'placeholder' option selectVoiceActivity.selectedIndex = 0; } async function populateAgentData() { try { const response = await fetch("/control/agent/info"); const data = await response.json(); agentIDInput.value = data.id; agentNameLabel.textContent = data.name; } catch (error) { console.error("Error populating agent data:", error); showError("Unable to load Agent data!") } } async function reloadAgentPrompt() { const agentId = agentIDInput.value; if (agentId !== undefined && agentId !== "") { await restartConversation(undefined, agentId); } else { showWarning("Cannot Re-Load Agent: No ID specified!") } } async function connect() { if(await setupWebRTC()) { if(await isOpenAIConnected()) { updateButtonState(); // update Connect/Disconnect button populateAIVoiceDropdown(); populateAgentData(); configureVADDropdown(); setElementDisabledState(false); } else { //stop webrtc if OpenAI connection was not established in time stopWebRTC(); } } } // HTML ELEMENT LISTENERS // add click event to start/stop conversation startButton.addEventListener('click', toggleConnection); // add mouse events to 'hold to talk' button talkButton.addEventListener("mousedown", () => toggleMicrophone(true)); talkButton.addEventListener("mouseup", () => toggleMicrophone(false)); talkButton.addEventListener("mouseleave", () => toggleMicrophone(false)); // add a change event to the voice dropdown to changet the AI voice selectAIVoice.addEventListener("change", updateSelectedAIVoice); // add a click event to the speaker button to toggle mute for browser layback speakerButton.addEventListener("click", toggleSpeaker) // add a click event to stop answer button stopAnswerButton.addEventListener("click", stopAnswer) // add a change event to change voice activity detection type selectVoiceActivity.addEventListener("change", updateVAD) // add a click event to restart conversation button restartConversationButton.addEventListener("click", () => restartConversation()) // add a click event to load a different agent prompt agentIDButton.addEventListener("click", reloadAgentPrompt) // call on page load and unload window.addEventListener("load", connect); window.addEventListener("beforeunload", stopWebRTC);