| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
|
|
| import { broadcastToVrm, type VrmBroadcastPayload } from './sse.js'; |
| import { getPrefs } from './prefs.js'; |
| import { stripForTts } from './text-utils.js'; |
| import { edgeTts, qwenTts, registerAudioFile, getAudioFile } from './tts.js'; |
| import { splitSentences } from './text-utils.js'; |
| import { SileroVad } from './voice/vad-service.js'; |
| import { HighPassFilter } from './voice/highpass-filter.js'; |
| import { readFileSync } from 'node:fs'; |
| import { buildVrmSystemPrompt } from '../skills/bundled/friendPrompt.js'; |
|
|
| |
|
|
| export type FriendServiceState = { |
| status: 'stopped' | 'starting' | 'running' | 'error'; |
| lastError?: string; |
| |
| displayClientCount?: number; |
| |
| captureStatus?: { capturing: boolean; interimText?: string }; |
| }; |
|
|
| type Listener = () => void; |
|
|
| |
| export type FriendInboundEvent = { |
| text: string; |
| }; |
|
|
| type InboundListener = (event: FriendInboundEvent) => void; |
|
|
| |
|
|
| type AudioCaptureProvider = { |
| startRecording( |
| onData: (chunk: Buffer) => void, |
| onEnd: () => void, |
| ): Promise<boolean>; |
| stopRecording(): Promise<void>; |
| isRecording(): boolean; |
| }; |
|
|
| |
|
|
| class FriendService { |
| private listeners = new Set<Listener>(); |
| private inboundListeners = new Set<InboundListener>(); |
| private state: FriendServiceState = { status: 'stopped' }; |
| |
| private capturing = false; |
| |
| private captureTranscripts: string[] = []; |
| |
| private captureInterimText = ''; |
| |
| private captureResolver: ((text: string) => void) | null = null; |
| |
| private audioCapture: AudioCaptureProvider | null = null; |
| |
| private sttConnection: { send: (chunk: Buffer) => void; finalize: () => Promise<void>; close: () => void } | null = null; |
| |
| private captureProvider = ''; |
| private captureLanguage = ''; |
| |
| private _flushing = false; |
| |
| private vadInstance: SileroVad | null = null; |
| |
| private highpassFilter = new HighPassFilter(80); |
| |
| private muted = false; |
| |
| private muteTimer: ReturnType<typeof setTimeout> | null = null; |
|
|
| |
|
|
| subscribe(listener: Listener): () => void { |
| this.listeners.add(listener); |
| return () => this.listeners.delete(listener); |
| } |
|
|
| subscribeToInbound(listener: InboundListener): () => void { |
| this.inboundListeners.add(listener); |
| return () => this.inboundListeners.delete(listener); |
| } |
|
|
| getStateSnapshot(): FriendServiceState { |
| return this.state; |
| } |
|
|
| |
|
|
| async start(): Promise<void> { |
| if (this.state.status === 'running') return; |
|
|
| this.setState({ status: 'starting', lastError: undefined }); |
|
|
| try { |
| |
| if (!this.vadInstance) { |
| const vad = new SileroVad({ |
| onSpeechStart: () => {}, |
| onSpeechEnd: (_audio) => { |
| this._flushVadSegment().catch((e) => |
| console.error('[FriendService] VAD segment flush error:', e), |
| ); |
| }, |
| }, { |
| |
| positiveSpeechThreshold: 0.90, |
| negativeSpeechThreshold: 0.40, |
| preSpeechTriggerFrames: 16, |
| minSpeechFrames: 8, |
| redemptionFrames: 15, |
| rmsThreshold: 0.015, |
| }); |
| vad.init().then(() => { |
| this.vadInstance = vad; |
| }).catch((e) => { |
| console.warn('[FriendService] VAD init failed (non-fatal, voice capture falls back to F2-only):', e); |
| }); |
| } |
|
|
| this.setState({ status: 'running' }); |
| } catch (err) { |
| const msg = err instanceof Error ? err.message : String(err); |
| this.setState({ status: 'error', lastError: msg }); |
| throw err; |
| } |
| } |
|
|
| async stop(): Promise<void> { |
| |
| if (this.capturing) { |
| await this.stopVoiceCapture().catch(() => {}); |
| } |
|
|
| this.audioCapture = null; |
| this.sttConnection = null; |
|
|
| if (this.state.status !== 'stopped') { |
| this.setState({ status: 'stopped' }); |
| } |
| } |
|
|
| |
|
|
| |
| |
| |
| |
| sendText(text: string): void { |
| const trimmed = text.trim(); |
| if (!trimmed) return; |
|
|
| |
| for (const listener of this.inboundListeners) { |
| listener({ text: trimmed }); |
| } |
|
|
| |
| const baseVrmPrompt = buildVrmSystemPrompt(); |
| const overrideSystemPrompt = `${baseVrmPrompt}\n\nIMPORTANT: You are ONLY the character(s) defined above. Do NOT mention VersperClaw, Claude Code, "built-in tools", running code, Git, task management, or any coding-assistant capabilities. You may use available tools when appropriate, but your identity and behavior must follow your character persona strictly.`; |
|
|
| |
| import('../utils/messageQueueManager.js').then(({ enqueue }) => { |
| enqueue({ |
| value: trimmed, |
| mode: 'prompt', |
| skipSlashCommands: true, |
| bridgeOrigin: true, |
| origin: { kind: 'channel', server: 'friend' }, |
| overrideSystemPrompt, |
| }); |
| }).catch((err) => { |
| console.error('[FriendService] enqueue failed:', err); |
| }); |
| } |
|
|
| |
|
|
| |
| |
| |
| |
| |
| |
| async startVoiceCapture(): Promise<void> { |
| if (this.capturing) return; |
|
|
| const prefs = getPrefs(); |
| let provider = prefs.sttProvider; |
| const language = prefs.sttLanguage || 'zh'; |
|
|
| |
| if (!provider || provider === 'browser') { |
| provider = await this.detectAvailableSttProvider(); |
| } |
|
|
| this.captureTranscripts = []; |
| this.captureInterimText = ''; |
| this.capturing = true; |
|
|
| try { |
| |
| |
| await this.withTimeout( |
| this._initVoiceCapture(provider, language), |
| 12000, |
| `Voice initialization timed out. Check that your microphone is accessible and STT provider "${provider}" is configured correctly.`, |
| ); |
| } catch (err) { |
| this.capturing = false; |
| this.sttConnection?.close(); |
| this.sttConnection = null; |
| throw err; |
| } |
| } |
|
|
| |
| |
| |
| |
| private async _initVoiceCapture( |
| provider: string, |
| language: string, |
| ): Promise<void> { |
| this.captureProvider = provider; |
| this.captureLanguage = language; |
|
|
| |
| const conn = await this.startSttConnectionWithTimeout(provider, language); |
| this.sttConnection = conn; |
|
|
| |
| const audio = await this.loadAudioCapture(); |
|
|
| |
| const ok = await audio.startRecording( |
| (chunk: Buffer) => { |
| this.sttConnection?.send(chunk); |
| }, |
| () => { |
| |
| }, |
| ); |
|
|
| if (!ok) { |
| throw new Error('Native audio capture unavailable'); |
| } |
|
|
| |
| |
| if (this.vadInstance) { |
| try { |
| this.highpassFilter.reset(); |
| this.vadInstance.start(); |
| } catch (e) { |
| console.warn('[FriendService] VAD start error (non-fatal):', e); |
| } |
| } |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| private async _flushVadSegment(): Promise<void> { |
| if (this._flushing || !this.capturing) return; |
| this._flushing = true; |
| try { |
| const oldConn = this.sttConnection; |
| if (!oldConn) return; |
|
|
| |
| const newConn = await this.startSttConnectionWithTimeout( |
| this.captureProvider, |
| this.captureLanguage, |
| ); |
| this.sttConnection = newConn; |
|
|
| |
| await oldConn.finalize().catch(() => {}); |
| oldConn.close(); |
|
|
| |
| const transcript = this.captureTranscripts.join('').trim(); |
| if (transcript) { |
| this.captureTranscripts = []; |
| this.sendText(transcript); |
|
|
| |
| |
| |
| |
| this.startAiTurnMute(); |
| } |
| } catch (err) { |
| console.error('[FriendService] _flushVadSegment error:', err); |
| } finally { |
| this._flushing = false; |
| } |
| } |
|
|
| |
| private async withTimeout<T>( |
| promise: Promise<T>, |
| ms: number, |
| message: string, |
| ): Promise<T> { |
| return Promise.race([ |
| promise, |
| new Promise<never>((_, reject) => |
| setTimeout(() => reject(new Error(message)), ms), |
| ), |
| ]); |
| } |
|
|
| |
| |
| |
| |
| private async detectAvailableSttProvider(): Promise<string> { |
| console.log('[FriendService] detectAvailableSttProvider: checking available providers...'); |
|
|
| |
| |
| try { |
| const { isGroqAvailable } = await import('../services/voice/groqSTT.js'); |
| if (isGroqAvailable()) { |
| console.log('[FriendService] detectAvailableSttProvider: Groq API key found'); |
| return 'groq'; |
| } |
| } catch { } |
|
|
| |
| try { |
| const { checkLocalWhisperAvailable } = await import( |
| '../services/voice/whisperSTT.js' |
| ); |
| const avail = await checkLocalWhisperAvailable(); |
| console.log('[FriendService] detectAvailableSttProvider: local Whisper available:', avail); |
| if (avail) { |
| return 'local'; |
| } |
| } catch (e) { |
| console.warn('[FriendService] detectAvailableSttProvider: local whisper check failed:', e); |
| } |
|
|
| |
| try { |
| const { isVoiceStreamAvailable } = await import( |
| '../services/voiceStreamSTT.js' |
| ); |
| if (isVoiceStreamAvailable()) { |
| return 'anthropic'; |
| } |
| } catch { } |
|
|
| |
| try { |
| const path = await import('node:path'); |
| const fs = await import('node:fs'); |
| const homeDir = process.env.HOME || process.env.USERPROFILE || ''; |
| const credsPath = path.join(homeDir, '.claude', 'tts', 'doubao', 'credentials.json'); |
| if (fs.existsSync(credsPath)) { |
| return 'doubao'; |
| } |
| } catch { } |
|
|
| throw new Error( |
| 'No STT provider available. Install local Whisper:\n' + |
| ' pip install openai-whisper\n\n' + |
| 'Or configure an STT provider in Friend settings (Settings β STT Provider).', |
| ); |
| } |
|
|
| |
| |
| |
| |
| private async startSttConnectionWithTimeout( |
| provider: string, |
| language: string, |
| ): Promise<{ send: (chunk: Buffer) => void; finalize: () => Promise<void>; close: () => void }> { |
| const timeoutMs = 8000; |
| const result = await Promise.race([ |
| this.startSttConnection(provider, language), |
| new Promise<never>((_, reject) => |
| setTimeout( |
| () => |
| reject( |
| new Error( |
| `STT provider "${provider}" timed out after ${timeoutMs / 1000}s.` + |
| (provider === 'local' |
| ? '\nInstall local Whisper: pip install openai-whisper' |
| : ''), |
| ), |
| ), |
| timeoutMs, |
| ), |
| ), |
| ]); |
| return result; |
| } |
|
|
| |
| |
| |
| async stopVoiceCapture(): Promise<string> { |
| return this._stopCapture(); |
| } |
|
|
| |
| |
| |
| private async _stopCapture(): Promise<string> { |
| console.log(`[FriendService] _stopCapture: capturing=${this.capturing} sttConnection=${this.sttConnection ? 'exists' : 'null'}`); |
| if (!this.capturing) return ''; |
|
|
| |
| if (this.audioCapture) { |
| await this.audioCapture.stopRecording().catch(() => {}); |
| } |
|
|
| this.capturing = false; |
|
|
| |
| this.clearMute(); |
|
|
| |
| if (this.vadInstance) { |
| try { |
| this.vadInstance.reset(); |
| } catch { } |
| } |
|
|
| |
| const conn = this.sttConnection; |
| this.sttConnection = null; |
|
|
| if (conn) { |
| try { |
| await conn.finalize(); |
| conn.close(); |
| } catch { |
| |
| } |
| } |
|
|
| |
| const remaining = this.captureTranscripts.join('').trim(); |
| if (remaining) { |
| this.sendText(remaining); |
| } |
|
|
| const transcript = this.captureTranscripts.join(''); |
| console.log(`[FriendService] _stopCapture: transcript="${transcript}" (len=${transcript.length})`); |
| this.captureTranscripts = []; |
| this.captureInterimText = ''; |
| this.setState({ captureStatus: { capturing: false } }); |
| return transcript; |
| } |
|
|
| |
| |
| |
| getCaptureStatus(): { capturing: boolean; interimText?: string } { |
| const status = this.state.captureStatus ?? { capturing: false }; |
| return { ...status }; |
| } |
|
|
| |
| |
| |
| |
| |
| |
| async transcribeAudioSegment(audioBuffer: Buffer): Promise<string> { |
| const prefs = getPrefs(); |
| let provider = prefs.sttProvider; |
| if (!provider || provider === 'browser') { |
| provider = await this.detectAvailableSttProvider(); |
| } |
|
|
| |
| const conn = await this.startSttConnectionWithTimeout( |
| provider, |
| prefs.sttLanguage || 'zh', |
| ); |
|
|
| try { |
| conn.send(audioBuffer); |
| await conn.finalize(); |
| conn.close(); |
| } catch (err) { |
| conn.close(); |
| throw err; |
| } |
|
|
| |
| const transcript = this.captureTranscripts.join(''); |
| this.captureTranscripts = []; |
|
|
| if (transcript.trim()) { |
| this.sendText(transcript); |
| } |
|
|
| return transcript; |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| private startAiTurnMute(): void { |
| if (!this.capturing) return; |
|
|
| |
| if (this.muteTimer) clearTimeout(this.muteTimer); |
|
|
| this.muted = true; |
| this.vadInstance?.pause(); |
|
|
| |
| |
| this.muteTimer = setTimeout(() => this.unmute(), 30_000); |
| } |
|
|
| |
| |
| |
| |
| |
| private extendMuteForTts(audioId: string): void { |
| if (!this.capturing) return; |
| if (!this.muted) return; |
|
|
| |
| if (this.muteTimer) clearTimeout(this.muteTimer); |
|
|
| |
| let muteMs = this.getMp3DurationMs(audioId); |
| if (muteMs <= 0) muteMs = 3000; |
|
|
| this.muteTimer = setTimeout(() => this.unmute(), muteMs); |
| } |
|
|
| |
| private unmute(): void { |
| this.muted = false; |
| this.muteTimer = null; |
| this.vadInstance?.start(); |
| } |
|
|
| |
| |
| |
| |
| |
| |
| private getMp3DurationMs(audioId: string): number { |
| const filePath = getAudioFile(audioId); |
| if (!filePath) return 0; |
| let buf: Buffer; |
| try { buf = readFileSync(filePath); } catch { return 0; } |
| if (buf.length < 100) return 0; |
|
|
| const isSync = (p: number) => |
| p + 1 < buf.length && buf[p] === 0xff && (buf[p + 1] & 0xe0) === 0xe0; |
|
|
| let offset = 0; |
|
|
| |
| if (buf[0] === 0x49 && buf[1] === 0x44 && buf[2] === 0x33) { |
| offset = 10 + |
| ((buf[6] & 0x7f) << 21) | |
| ((buf[7] & 0x7f) << 14) | |
| ((buf[8] & 0x7f) << 7) | |
| (buf[9] & 0x7f); |
| } |
|
|
| |
| let firstSync = -1; |
| let secondSync = -1; |
| for (let i = offset; i < buf.length - 3; i++) { |
| if (isSync(i)) { |
| if (firstSync === -1) firstSync = i; |
| else { secondSync = i; break; } |
| } |
| } |
| if (firstSync === -1 || secondSync === -1) return 0; |
|
|
| const frameSize = secondSync - firstSync; |
| if (frameSize < 20) return 0; |
|
|
| |
| const h = |
| (buf[firstSync] << 24) | |
| (buf[firstSync + 1] << 16) | |
| (buf[firstSync + 2] << 8) | |
| buf[firstSync + 3]; |
| const version = (h >> 19) & 0x3; |
| const sampleRateIdx = (h >> 10) & 0x3; |
| if (sampleRateIdx === 3) return 0; |
|
|
| const srTable: Record<number, number> = { |
| 3: [44100, 48000, 32000][sampleRateIdx], |
| 2: [22050, 24000, 16000][sampleRateIdx], |
| 0: [11025, 12000, 8000][sampleRateIdx], |
| }; |
| const sampleRate = srTable[version]; |
| if (!sampleRate) return 0; |
|
|
| const isMpeg1 = version === 3; |
| const spf = isMpeg1 ? 1152 : 576; |
|
|
| |
| let frames = 0; |
| for (let pos = firstSync; pos + 3 < buf.length; pos += frameSize) { |
| |
| if (!isSync(pos)) { |
| |
| while (pos < buf.length - 3 && !isSync(pos)) pos++; |
| if (pos >= buf.length - 3) break; |
| } |
| frames++; |
| } |
|
|
| return Math.round((frames * spf) / sampleRate * 1000); |
| } |
|
|
| |
| private clearMute(): void { |
| if (this.muteTimer) { |
| clearTimeout(this.muteTimer); |
| this.muteTimer = null; |
| } |
| this.muted = false; |
| } |
|
|
| |
|
|
| |
| |
| |
| |
| |
| async broadcastResponse(text: string): Promise<void> { |
| if (!text.trim()) return; |
|
|
| const prefs = getPrefs(); |
|
|
| |
| broadcastToVrm({ text }); |
|
|
| |
| if (prefs.ttsEnabled) { |
| try { |
| const audioId = await this.generateTts(text); |
| if (audioId) { |
| const fullUrl = `http://127.0.0.1:3456/plugins/friend/audio/${audioId}`; |
| broadcastToVrm({ audioUrl: fullUrl, sendFirstTts: true }); |
|
|
| |
| this.extendMuteForTts(audioId); |
| } |
| } catch (err) { |
| console.warn('[FriendService] TTS generation failed:', err); |
| } |
| } |
|
|
| |
| broadcastToVrm({ replyDone: true }); |
| } |
|
|
| |
| |
| |
| broadcastVrm(payload: VrmBroadcastPayload): void { |
| broadcastToVrm(payload); |
| } |
|
|
| |
|
|
| private async startSttConnection( |
| provider: string, |
| language: string, |
| ): Promise<{ send: (chunk: Buffer) => void; finalize: () => Promise<void>; close: () => void }> { |
| const callbacks = { |
| onTranscript: (text: string, isFinal: boolean) => { |
| if (isFinal) { |
| this.captureTranscripts.push(text); |
| this.captureInterimText = ''; |
| } else { |
| this.captureInterimText = text; |
| } |
| |
| this.setState({ |
| captureStatus: { capturing: true, interimText: this.captureInterimText }, |
| }); |
| }, |
| onError: (_error: string) => {}, |
| onClose: () => {}, |
| onReady: (_conn: any) => {}, |
| }; |
|
|
| switch (provider) { |
| case 'anthropic': { |
| const { connectVoiceStream, isVoiceStreamAvailable } = await import( |
| '../services/voiceStreamSTT.js' |
| ); |
| if (!isVoiceStreamAvailable()) { |
| throw new Error('Anthropic Voice Stream not available'); |
| } |
| return await connectVoiceStream(callbacks, { language, keyterms: ['code', 'versperclaw'] }); |
| } |
|
|
| case 'local': { |
| const { connectLocalWhisperStream, preloadWhisperModel } = await import( |
| '../services/voice/whisperSTT.js' |
| ); |
| await preloadWhisperModel({ language }); |
| return await connectLocalWhisperStream(callbacks, { language }); |
| } |
|
|
| case 'doubao': { |
| const { connectDoubaoStream } = await import('../services/doubaoSTT.js'); |
| return await connectDoubaoStream(callbacks, { language: language || 'zh' }); |
| } |
|
|
| case 'groq': { |
| const { connectGroqStream } = await import( |
| '../services/voice/groqSTT.js' |
| ); |
| return await connectGroqStream(callbacks, { language }); |
| } |
|
|
| default: |
| throw new Error(`Unknown STT provider: ${provider}`); |
| } |
| } |
|
|
| |
|
|
| private async loadAudioCapture(): Promise<AudioCaptureProvider> { |
| if (this.audioCapture) return this.audioCapture; |
|
|
| |
| |
| |
| |
| const { spawn } = await import('node:child_process'); |
| let captureProc: import('node:child_process').ChildProcess | null = null; |
|
|
| this.audioCapture = { |
| startRecording: async (onData, _onEnd) => { |
| for (const tool of ['arecord', 'parecord']) { |
| try { |
| const args = tool === 'arecord' |
| ? ['-D', 'default', '-r', '16000', '-f', 'S16_LE', '-c', '1', '-t', 'raw', '-q'] |
| : ['--raw', '--rate=16000', '--format=s16le', '--channels=1', '--latency-msec=20']; |
| const proc = spawn(tool, args, { stdio: ['pipe', 'pipe', 'pipe'] }); |
| if (proc.pid === undefined) continue; |
|
|
| |
| |
| |
| let dataArrived = false; |
| let verifyTimer: ReturnType<typeof setTimeout> | null = null; |
|
|
| const verified = await new Promise<boolean>((resolve) => { |
| const feedAudio = (c: Buffer) => { |
| |
| if (this.muted) return; |
|
|
| |
| onData(c); |
|
|
| |
| if (this.vadInstance) { |
| const float32 = new Float32Array(c.length / 2); |
| for (let i = 0; i < float32.length; i++) { |
| float32[i] = c.readInt16LE(i * 2) / 32768; |
| } |
| |
| this.highpassFilter.process(float32); |
| this.vadInstance.processAudio(float32).catch(() => {}); |
| } |
| }; |
|
|
| const dataHandler = (chunk: Buffer) => { |
| dataArrived = true; |
| if (verifyTimer) { clearTimeout(verifyTimer); } |
| feedAudio(chunk); |
| |
| proc.stdout?.removeListener('data', dataHandler); |
| proc.stdout?.on('data', feedAudio); |
| resolve(true); |
| }; |
| proc.stdout?.on('data', dataHandler); |
|
|
| |
| proc.on('exit', () => { |
| if (!dataArrived) { |
| if (verifyTimer) { clearTimeout(verifyTimer); } |
| resolve(false); |
| } |
| }); |
|
|
| |
| verifyTimer = setTimeout(() => { |
| if (!dataArrived) resolve(false); |
| }, 500); |
| }); |
|
|
| if (verified) { |
| captureProc = proc; |
| proc.on('exit', () => { captureProc = null; }); |
| return true; |
| } |
|
|
| |
| proc.kill('SIGTERM'); |
| } catch { |
| continue; |
| } |
| } |
| return false; |
| }, |
| stopRecording: async () => { |
| if (captureProc) { |
| captureProc.kill('SIGTERM'); |
| setTimeout(() => { |
| try { captureProc?.kill('SIGKILL'); } catch {} |
| }, 2000); |
| captureProc = null; |
| } |
| }, |
| isRecording: () => captureProc !== null, |
| }; |
|
|
| return this.audioCapture; |
| } |
|
|
| |
|
|
| private async generateTts(text: string): Promise<string | undefined> { |
| const prefs = getPrefs(); |
| if (!prefs.ttsEnabled) return undefined; |
|
|
| const cleanText = stripForTts(text); |
| if (!cleanText) return undefined; |
|
|
| let result: { success: boolean; audioPath?: string; error?: string }; |
| if (prefs.provider === 'qwen' && prefs.qwenKey) { |
| result = await qwenTts({ |
| text: cleanText, |
| apiKey: prefs.qwenKey, |
| voice: prefs.voice, |
| model: prefs.qwenModel, |
| language: prefs.language, |
| }); |
| } else { |
| result = await edgeTts({ text: cleanText, voice: prefs.voice }); |
| } |
|
|
| if (result.success && result.audioPath) { |
| return registerAudioFile(result.audioPath); |
| } |
|
|
| return undefined; |
| } |
|
|
| |
|
|
| private setState(next: Partial<FriendServiceState>): void { |
| this.state = { ...this.state, ...next }; |
| for (const listener of this.listeners) listener(); |
| } |
| } |
|
|
| |
| export const friendService = new FriendService(); |
|
|