Merge pull request sugarforever#596 from sugarforever/feature/openai-…

…realtime-webrtc

Feature/openai realtime webrtc

components/IconMicrophone.vue

@@ -0,0 +1,5 @@
+<template>
+    <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
+        <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 11a7 7 0 01-7 7m0 0a7 7 0 01-7-7m7 7v4m0 0H8m4 0h4m-4-8a3 3 0 01-3-3V5a3 3 0 116 0v6a3 3 0 01-3 3z" />
+    </svg>
+</template>

components/IconSpinner.vue

@@ -0,0 +1,5 @@
+<template>
+  <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
+    <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15" />
+  </svg>
+</template>

components/IconStop.vue

@@ -0,0 +1,6 @@
+<template>
+    <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
+        <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
+        <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 10a1 1 0 011-1h4a1 1 0 011 1v4a1 1 0 01-1 1h-4a1 1 0 01-1-1v-4z" />
+    </svg>
+</template>

pages/realtime/index.vue

@@ -0,0 +1,185 @@
+<template>
+  <div class="flex items-center justify-center">
+    <div class="max-w-md w-full p-6 bg-white dark:bg-gray-800 rounded-lg shadow-lg">
+      <h1 class="text-2xl font-bold mb-10 text-center dark:text-gray-100">Realtime WebRTC Connection</h1>
+
+      <div class="space-y-6">
+        <!-- Connection status and button combined -->
+        <div class="flex flex-col items-center">
+          <button
+                  @click="handleConnectionToggle"
+                  class="w-16 h-16 rounded-full flex items-center justify-center transition-all duration-300 relative"
+                  :class="buttonStyles"
+                  :disabled="connectionStatus === 'connecting'">
+            <!-- Different icons based on status -->
+            <IconMicrophone v-if="connectionStatus === 'disconnected'" class="w-8 h-8" />
+            <IconSpinner v-else-if="connectionStatus === 'connecting'" class="w-8 h-8 animate-spin" />
+            <IconStop v-else class="w-8 h-8" />
+
+            <!-- Ripple effect when active -->
+            <div v-if="connectionStatus === 'connected'" class="absolute inset-0">
+              <div class="absolute inset-0 rounded-full animate-ping opacity-25 bg-primary-500 dark:bg-primary-700"></div>
+            </div>
+          </button>
+
+          <span class="mt-6 text-sm text-gray-600 dark:text-gray-400">{{ statusText }}</span>
+        </div>
+
+        <!-- Audio wave visualization when connected -->
+        <div v-if="connectionStatus === 'connected'" class="flex justify-center items-center h-12">
+          <div v-for="n in 5" :key="n"
+               class="mx-1 w-1 bg-primary-500 dark:bg-primary-700 rounded-full animate-wave"
+               :style="`height: ${20 + Math.random() * 20}px; animation-delay: ${n * 0.1}s`">
+          </div>
+        </div>
+
+        <!-- Error message -->
+        <div v-if="error" class="text-red-500 dark:text-red-400 text-center text-sm">
+          {{ error }}
+        </div>
+      </div>
+    </div>
+  </div>
+</template>
+
+<script setup>
+import { getKeysHeader } from '~/utils/settings'
+
+const connectionStatus = ref('disconnected')
+const error = ref('')
+const peerConnection = ref(null)
+const dataChannel = ref(null)
+
+// Computed properties for UI
+const buttonStyles = computed(() => ({
+  'bg-primary-500 hover:bg-primary-600 dark:bg-primary-700 dark:hover:bg-primary-800 text-white':
+    connectionStatus.value === 'disconnected',
+  'bg-yellow-500 dark:bg-yellow-600 cursor-not-allowed':
+    connectionStatus.value === 'connecting',
+  'bg-red-500 hover:bg-red-600 dark:bg-red-700 dark:hover:bg-red-800 text-white':
+    connectionStatus.value === 'connected'
+}))
+
+const statusText = computed(() => {
+  switch (connectionStatus.value) {
+    case 'connecting':
+      return 'Initializing connection...'
+    case 'connected':
+      return 'Tap to end conversation'
+    default:
+      return 'Tap to start conversation'
+  }
+})
+
+// Handle connection toggle
+async function handleConnectionToggle() {
+  if (connectionStatus.value === 'connected') {
+    await stopConnection()
+  } else {
+    await initConnection()
+  }
+}
+
+// Stop connection
+async function stopConnection() {
+  if (peerConnection.value) {
+    peerConnection.value.close()
+    peerConnection.value = null
+  }
+  if (dataChannel.value) {
+    dataChannel.value.close()
+    dataChannel.value = null
+  }
+  connectionStatus.value = 'disconnected'
+}
+
+async function initConnection() {
+  try {
+    connectionStatus.value = 'connecting'
+    error.value = ''
+
+    // Get ephemeral token from server with proper headers
+    const tokenResponse = await fetch('/api/audio/session', {
+      method: 'POST',
+      headers: getKeysHeader()
+    })
+    const data = await tokenResponse.json()
+    const EPHEMERAL_KEY = data.client_secret.value
+
+    // Create peer connection
+    peerConnection.value = new RTCPeerConnection()
+
+    // Set up audio element for remote audio
+    const audioEl = document.createElement('audio')
+    audioEl.autoplay = true
+    peerConnection.value.ontrack = e => audioEl.srcObject = e.streams[0]
+
+    // Request microphone access and add local track
+    const mediaStream = await navigator.mediaDevices.getUserMedia({
+      audio: true
+    })
+    peerConnection.value.addTrack(mediaStream.getTracks()[0])
+
+    // Setup data channel
+    dataChannel.value = peerConnection.value.createDataChannel('oai-events')
+    dataChannel.value.addEventListener('message', (e) => {
+      console.log('Received message:', e)
+    })
+
+    // Create and set local description
+    const offer = await peerConnection.value.createOffer()
+    await peerConnection.value.setLocalDescription(offer)
+
+    // Connect to OpenAI realtime API
+    const baseUrl = 'https://api.openai.com/v1/realtime'
+    const model = 'gpt-4o-realtime-preview-2024-12-17'
+    const sdpResponse = await fetch(`${baseUrl}?model=${model}`, {
+      method: 'POST',
+      body: offer.sdp,
+      headers: {
+        Authorization: `Bearer ${EPHEMERAL_KEY}`,
+        'Content-Type': 'application/sdp'
+      },
+    })
+
+    const answer = {
+      type: 'answer',
+      sdp: await sdpResponse.text(),
+    }
+    await peerConnection.value.setRemoteDescription(answer)
+
+    // Update connection status
+    connectionStatus.value = 'connected'
+
+    // Handle connection state changes
+    peerConnection.value.onconnectionstatechange = () => {
+      if (peerConnection.value.connectionState === 'disconnected') {
+        connectionStatus.value = 'disconnected'
+      }
+    }
+
+  } catch (err) {
+    console.error('Connection error:', err)
+    error.value = `Failed to connect: ${err.message}`
+    connectionStatus.value = 'disconnected'
+  }
+}
+</script>
+
+<style scoped>
+@keyframes wave {
+
+  0%,
+  100% {
+    transform: scaleY(0.5);
+  }
+
+  50% {
+    transform: scaleY(1);
+  }
+}
+
+.animate-wave {
+  animation: wave 1s ease-in-out infinite;
+}
+</style>

server/api/audio/session.post.ts

@@ -0,0 +1,28 @@
+import { defineEventHandler } from 'h3'
+
+export default defineEventHandler(async (event) => {
+  try {
+    const apiKey = event.context.keys.openai.key || ''
+    const response = await fetch('https://api.openai.com/v1/realtime/sessions', {
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${apiKey}`,
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify({
+        model: 'gpt-4o-realtime-preview-2024-12-17',
+        voice: 'alloy'
+      })
+    })
+
+    const data = await response.json()
+    return data
+
+  } catch (error) {
+    console.error('Error creating audio session:', error)
+    throw createError({
+      statusCode: 500,
+      message: 'Failed to create audio session'
+    })
+  }
+})