chore(services/discord-bot): remote stt, fix server bugs

cspell.config.yaml

@@ -77,6 +77,7 @@ words:
   - opusscript
   - pgvector
   - picklist
+  - picovoice
   - pinia
   - pixi
   - pixiv
@@ -104,6 +105,7 @@ words:
   - wavefile
   - webgpu
   - worklet
+  - Xenova
   - xsai
 ignoreWords: []
 import: []

packages/server-runtime/src/index.ts

@@ -1,4 +1,5 @@
 import type { WebSocketEvent } from '@proj-airi/server-shared/types'
+import type { Peer } from 'crossws'
 import { Format, LogLevel, setGlobalFormat, setGlobalLogLevel, useLogg } from '@guiiai/logg'
 import { createApp, createRouter, defineWebSocketHandler } from 'h3'
 
@@ -15,24 +16,34 @@ export const app = createApp({
 const router = createRouter()
 app.use(router)
 
+const peers = new Set<Peer>()
+
 router.get('/ws', defineWebSocketHandler({
   open: (peer) => {
-    websocketLogger.withFields({ peer: peer.id }).log('connected')
+    peers.add(peer)
+    websocketLogger.withFields({ peer: peer.id, activePeers: peers.size }).log('connected')
   },
   message: (peer, message) => {
     const event = message.json() as WebSocketEvent
 
-    websocketLogger.withFields({ peer: peer.id, message: event }).log('received message')
     switch (event.type) {
+      case 'input:text':
+        break
       case 'input:text:voice':
-        websocketLogger.withFields({ message: event }).log('transcribed')
         break
     }
+
+    for (const p of peers) {
+      if (p.id !== peer.id) {
+        p.send(JSON.stringify(event))
+      }
+    }
   },
   error: (peer, error) => {
     websocketLogger.withFields({ peer: peer.id }).withError(error).error('an error occurred')
   },
   close: (peer, details) => {
-    websocketLogger.withFields({ peer: peer.id, details }).log('closed')
+    websocketLogger.withFields({ peer: peer.id, details, activePeers: peers.size }).log('closed')
+    peers.delete(peer)
   },
 }))

packages/server-sdk/src/client.ts

@@ -1,4 +1,4 @@
-import type { WebSocketEvent, WebSocketEvents } from '@proj-airi/server-shared/types'
+import type { WebSocketBaseEvent, WebSocketEvent, WebSocketEvents } from '@proj-airi/server-shared/types'
 import type { Blob } from 'node:buffer'
 import WebSocket from 'crossws/websocket'
 import { defu } from 'defu'
@@ -11,18 +11,43 @@ export interface ClientOptions {
 
 export class Client {
   private websocket: WebSocket
+  private eventListeners: Map<keyof WebSocketEvents, Array<(data: WebSocketBaseEvent<keyof WebSocketEvents, WebSocketEvents[keyof WebSocketEvents]>) => void | Promise<void>>> = new Map()
 
   constructor(options: ClientOptions) {
     const opts = defu<Required<ClientOptions>, Required<Omit<ClientOptions, 'name'>>[]>(options, { url: 'ws://localhost:6121/ws', possibleEvents: [] })
 
     this.websocket = new WebSocket(opts.url)
-    this.send({
-      type: 'module:announce',
-      data: {
-        name: opts.name,
-        possibleEvents: opts.possibleEvents,
-      },
-    })
+    this.websocket.onmessage = this.handleMessage.bind(this)
+    this.websocket.onopen = () => {
+      this.send({
+        type: 'module:announce',
+        data: {
+          name: opts.name,
+          possibleEvents: opts.possibleEvents,
+        },
+      })
+    }
+  }
+
+  private async handleMessage(event: any) {
+    const data = JSON.parse(event.data) as WebSocketEvent
+    const listeners = this.eventListeners.get(data.type)
+    if (!listeners)
+      return
+
+    for (const listener of listeners) {
+      await listener(data)
+    }
+  }
+
+  onEvent<E extends keyof WebSocketEvents>(event: E, callback: (data: WebSocketBaseEvent<E, WebSocketEvents[E]>) => void | Promise<void>): void {
+    if (!this.eventListeners.get(event)) {
+      this.eventListeners.set(event, [])
+    }
+
+    const listeners = this.eventListeners.get(event)
+    listeners.push(callback)
+    this.eventListeners.set(event, listeners)
   }
 
   send(data: WebSocketEvent): void {

services/discord-bot/.env

@@ -2,8 +2,11 @@ DISCORD_TOKEN=''
 DISCORD_BOT_CLIENT_ID=''
 
 OPENAI_MODEL=''
-OPENAI_API_KEY=''
 OPENAI_API_BASE_URL=''
+OPENAI_API_KEY=''
+
+OPENAI_STT_API_BASE_URL=''
+OPENAI_STT_API_KEY=''
 
 ELEVENLABS_API_KEY=''
 ELEVENLABS_API_BASE_URL=''

services/discord-bot/package.json

@@ -15,7 +15,7 @@
     "directory": "services/discord-bot"
   },
   "scripts": {
-    "start": "dotenvx run -f .env.local -f .env --ignore=MISSING_ENV_FILE -- tsx src/index.ts"
+    "start": "dotenvx run -f .env -f .env.local --overload --ignore=MISSING_ENV_FILE -- tsx src/index.ts"
   },
   "dependencies": {
     "@discordjs/voice": "^0.18.0",
@@ -26,6 +26,7 @@
     "@proj-airi/server-shared": "workspace:^",
     "@xsai/generate-speech": "catalog:",
     "@xsai/generate-text": "catalog:",
+    "@xsai/generate-transcription": "^0.0.28",
     "@xsai/providers": "catalog:",
     "@xsai/shared-chat": "catalog:",
     "discord.js": "^14.17.3",