FocusGuardBaseModel / src /utils /VideoManagerLocal.js
Kexin-251202's picture
Deploy base model
c86c45b verified
// src/utils/VideoManagerLocal.js
// 本地视频处理版本 - 使用 WebSocket + Canvas,不依赖 WebRTC
export class VideoManagerLocal {
constructor(callbacks) {
this.callbacks = callbacks || {};
this.localVideoElement = null; // 显示本地摄像头
this.displayVideoElement = null; // 显示处理后的视频
this.canvas = null;
this.stream = null;
this.ws = null;
this.isStreaming = false;
this.sessionId = null;
this.sessionStartTime = null;
this.frameRate = 15; // 降低帧率以减少网络负载
this.captureInterval = null;
// 状态平滑处理
this.currentStatus = false;
this.statusBuffer = [];
this.bufferSize = 3;
// 检测数据
this.latestDetectionData = null;
this.lastConfidence = 0;
// 通知系统
this.notificationEnabled = true;
this.notificationThreshold = 30;
this.unfocusedStartTime = null;
this.lastNotificationTime = null;
this.notificationCooldown = 60000;
// 性能统计
this.stats = {
framesSent: 0,
framesProcessed: 0,
avgLatency: 0,
lastLatencies: []
};
}
// 初始化摄像头
async initCamera(localVideoRef, displayCanvasRef) {
try {
console.log('Initializing local camera...');
this.stream = await navigator.mediaDevices.getUserMedia({
video: {
width: { ideal: 640 },
height: { ideal: 480 },
facingMode: 'user'
},
audio: false
});
this.localVideoElement = localVideoRef;
this.displayCanvas = displayCanvasRef;
// 显示本地视频流
if (this.localVideoElement) {
this.localVideoElement.srcObject = this.stream;
this.localVideoElement.play();
}
// 创建用于截图的 canvas
this.canvas = document.createElement('canvas');
this.canvas.width = 640;
this.canvas.height = 480;
console.log('Local camera initialized');
return true;
} catch (error) {
console.error('Camera init error:', error);
throw error;
}
}
// 开始流式处理
async startStreaming() {
if (!this.stream) {
throw new Error('Camera not initialized');
}
if (this.isStreaming) {
console.warn('Already streaming');
return;
}
console.log('Starting WebSocket streaming...');
this.isStreaming = true;
// 请求通知权限
await this.requestNotificationPermission();
await this.loadNotificationSettings();
// 建立 WebSocket 连接
await this.connectWebSocket();
// 开始定期截图并发送
this.startCapture();
console.log('Streaming started');
}
// 建立 WebSocket 连接
async connectWebSocket() {
return new Promise((resolve, reject) => {
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
const wsUrl = `${protocol}//${window.location.host}/ws/video`;
console.log('Connecting to WebSocket:', wsUrl);
this.ws = new WebSocket(wsUrl);
this.ws.onopen = () => {
console.log('WebSocket connected');
// 发送开始会话请求
this.ws.send(JSON.stringify({ type: 'start_session' }));
resolve();
};
this.ws.onmessage = (event) => {
try {
const data = JSON.parse(event.data);
this.handleServerMessage(data);
} catch (e) {
console.error('Failed to parse message:', e);
}
};
this.ws.onerror = (error) => {
console.error('WebSocket error:', error);
reject(error);
};
this.ws.onclose = () => {
console.log('WebSocket disconnected');
if (this.isStreaming) {
console.log('Attempting to reconnect...');
setTimeout(() => this.connectWebSocket(), 2000);
}
};
});
}
// 开始截图并发送
startCapture() {
const interval = 1000 / this.frameRate; // 转换为毫秒
this.captureInterval = setInterval(() => {
if (!this.isStreaming || !this.ws || this.ws.readyState !== WebSocket.OPEN) {
return;
}
try {
// 从视频流截取一帧
const ctx = this.canvas.getContext('2d');
ctx.drawImage(this.localVideoElement, 0, 0, this.canvas.width, this.canvas.height);
// 转换为 JPEG base64(压缩质量 0.8)
const imageData = this.canvas.toDataURL('image/jpeg', 0.8);
const base64Data = imageData.split(',')[1];
// 发送到服务器
const timestamp = Date.now();
this.ws.send(JSON.stringify({
type: 'frame',
image: base64Data,
timestamp: timestamp
}));
this.stats.framesSent++;
} catch (error) {
console.error('Capture error:', error);
}
}, interval);
console.log(`Capturing at ${this.frameRate} FPS`);
}
// 处理服务器消息
handleServerMessage(data) {
switch (data.type) {
case 'session_started':
this.sessionId = data.session_id;
this.sessionStartTime = Date.now();
console.log('Session started:', this.sessionId);
if (this.callbacks.onSessionStart) {
this.callbacks.onSessionStart(this.sessionId);
}
break;
case 'detection':
this.stats.framesProcessed++;
// 计算延迟
if (data.timestamp) {
const latency = Date.now() - data.timestamp;
this.stats.lastLatencies.push(latency);
if (this.stats.lastLatencies.length > 10) {
this.stats.lastLatencies.shift();
}
this.stats.avgLatency =
this.stats.lastLatencies.reduce((a, b) => a + b, 0) /
this.stats.lastLatencies.length;
}
// 更新状态
this.updateStatus(data.focused);
this.latestDetectionData = {
detections: data.detections || [],
confidence: data.confidence || 0,
focused: data.focused,
timestamp: performance.now()
};
this.lastConfidence = data.confidence || 0;
if (this.callbacks.onStatusUpdate) {
this.callbacks.onStatusUpdate(this.currentStatus);
}
// 在 display canvas 上绘制结果
this.drawDetectionResult(data);
break;
case 'session_ended':
console.log('Received session_ended message');
console.log('Session summary:', data.summary);
if (this.callbacks.onSessionEnd) {
console.log('Calling onSessionEnd callback');
this.callbacks.onSessionEnd(data.summary);
} else {
console.warn('No onSessionEnd callback registered');
}
this.sessionId = null;
this.sessionStartTime = null;
break;
case 'error':
console.error('Server error:', data.message);
break;
default:
console.log('Unknown message type:', data.type);
}
}
// 在 canvas 上绘制检测结果
drawDetectionResult(data) {
if (!this.displayCanvas) return;
const ctx = this.displayCanvas.getContext('2d');
// 绘制当前帧
ctx.drawImage(this.localVideoElement, 0, 0, this.displayCanvas.width, this.displayCanvas.height);
// 绘制检测框
if (data.detections && data.detections.length > 0) {
data.detections.forEach(det => {
const [x1, y1, x2, y2] = det.bbox;
const color = data.focused ? '#00FF00' : '#FF0000';
// 绘制边框
ctx.strokeStyle = color;
ctx.lineWidth = 3;
ctx.strokeRect(x1, y1, x2 - x1, y2 - y1);
// 绘制标签
const label = `${det.class_name || 'person'} ${(det.confidence * 100).toFixed(1)}%`;
ctx.fillStyle = color;
ctx.font = '16px Arial';
ctx.fillText(label, x1, Math.max(20, y1 - 5));
});
}
// 绘制状态文字
const statusText = data.focused ? 'FOCUSED' : 'NOT FOCUSED';
const color = data.focused ? '#00FF00' : '#FF0000';
ctx.fillStyle = color;
ctx.font = 'bold 24px Arial';
ctx.fillText(statusText, 10, 30);
ctx.font = '16px Arial';
ctx.fillText(`Confidence: ${(data.confidence * 100).toFixed(1)}%`, 10, 55);
// 显示性能统计
ctx.font = '12px Arial';
ctx.fillStyle = '#FFFFFF';
ctx.fillText(`FPS: ${this.frameRate} | Latency: ${this.stats.avgLatency.toFixed(0)}ms`, 10, this.displayCanvas.height - 10);
}
updateStatus(newFocused) {
this.statusBuffer.push(newFocused);
if (this.statusBuffer.length > this.bufferSize) {
this.statusBuffer.shift();
}
if (this.statusBuffer.length < this.bufferSize) return false;
const focusedCount = this.statusBuffer.filter(f => f).length;
const focusedRatio = focusedCount / this.statusBuffer.length;
const previousStatus = this.currentStatus;
if (focusedRatio >= 0.75) {
this.currentStatus = true;
} else if (focusedRatio <= 0.25) {
this.currentStatus = false;
}
this.handleNotificationLogic(previousStatus, this.currentStatus);
}
handleNotificationLogic(previousStatus, currentStatus) {
const now = Date.now();
if (previousStatus && !currentStatus) {
this.unfocusedStartTime = now;
}
if (!previousStatus && currentStatus) {
this.unfocusedStartTime = null;
}
if (!currentStatus && this.unfocusedStartTime) {
const unfocusedDuration = (now - this.unfocusedStartTime) / 1000;
if (unfocusedDuration >= this.notificationThreshold) {
const canSendNotification = !this.lastNotificationTime ||
(now - this.lastNotificationTime) >= this.notificationCooldown;
if (canSendNotification) {
this.sendNotification(
'Focus Alert',
`You've been distracted for ${Math.floor(unfocusedDuration)} seconds. Get back to work!`
);
this.lastNotificationTime = now;
}
}
}
}
async requestNotificationPermission() {
if ('Notification' in window && Notification.permission === 'default') {
try {
await Notification.requestPermission();
} catch (error) {
console.error('Failed to request notification permission:', error);
}
}
}
async loadNotificationSettings() {
try {
const response = await fetch('/api/settings');
const settings = await response.json();
if (settings) {
this.notificationEnabled = settings.notification_enabled ?? true;
this.notificationThreshold = settings.notification_threshold ?? 30;
}
} catch (error) {
console.error('Failed to load notification settings:', error);
}
}
sendNotification(title, message) {
if (!this.notificationEnabled) return;
if ('Notification' in window && Notification.permission === 'granted') {
try {
const notification = new Notification(title, {
body: message,
icon: '/vite.svg',
badge: '/vite.svg',
tag: 'focus-guard-distraction',
requireInteraction: false
});
setTimeout(() => notification.close(), 3000);
} catch (error) {
console.error('Failed to send notification:', error);
}
}
}
async stopStreaming() {
console.log('Stopping streaming...');
this.isStreaming = false;
// 停止截图
if (this.captureInterval) {
clearInterval(this.captureInterval);
this.captureInterval = null;
}
// 发送结束会话请求并等待响应
if (this.ws && this.ws.readyState === WebSocket.OPEN && this.sessionId) {
const sessionId = this.sessionId;
// 等待 session_ended 消息
const waitForSessionEnd = new Promise((resolve) => {
const originalHandler = this.ws.onmessage;
const timeout = setTimeout(() => {
this.ws.onmessage = originalHandler;
console.log('Session end timeout, proceeding anyway');
resolve();
}, 2000);
this.ws.onmessage = (event) => {
try {
const data = JSON.parse(event.data);
if (data.type === 'session_ended') {
clearTimeout(timeout);
this.handleServerMessage(data);
this.ws.onmessage = originalHandler;
resolve();
} else {
// 仍然处理其他消息
this.handleServerMessage(data);
}
} catch (e) {
console.error('Failed to parse message:', e);
}
};
});
console.log('Sending end_session request for session:', sessionId);
this.ws.send(JSON.stringify({
type: 'end_session',
session_id: sessionId
}));
// 等待响应或超时
await waitForSessionEnd;
}
// 延迟关闭 WebSocket 确保消息发送完成
await new Promise(resolve => setTimeout(resolve, 200));
// 关闭 WebSocket
if (this.ws) {
this.ws.close();
this.ws = null;
}
// 停止摄像头
if (this.stream) {
this.stream.getTracks().forEach(track => track.stop());
this.stream = null;
}
// 清空视频
if (this.localVideoElement) {
this.localVideoElement.srcObject = null;
}
// 清空 canvas
if (this.displayCanvas) {
const ctx = this.displayCanvas.getContext('2d');
ctx.clearRect(0, 0, this.displayCanvas.width, this.displayCanvas.height);
}
// 清理状态
this.unfocusedStartTime = null;
this.lastNotificationTime = null;
console.log('Streaming stopped');
console.log('Stats:', this.stats);
}
setFrameRate(rate) {
this.frameRate = Math.max(1, Math.min(30, rate));
console.log(`Frame rate set to ${this.frameRate} FPS`);
// 重启截图(如果正在运行)
if (this.isStreaming && this.captureInterval) {
clearInterval(this.captureInterval);
this.startCapture();
}
}
getStats() {
return {
...this.stats,
isStreaming: this.isStreaming,
sessionId: this.sessionId,
currentStatus: this.currentStatus,
lastConfidence: this.lastConfidence
};
}
}