Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
k23099462 commited on
Commit ·
3dc18ac
1
Parent(s): 0d448a6
fix: eye gaze toggle now properly controls gaze line visibility
Browse files- .gitignore +1 -0
- main.py +11 -8
- src/utils/VideoManagerLocal.js +64 -59
.gitignore
CHANGED
|
@@ -45,3 +45,4 @@ test_focus_guard.db
|
|
| 45 |
__pycache__/
|
| 46 |
docs/
|
| 47 |
docs
|
|
|
|
|
|
| 45 |
__pycache__/
|
| 46 |
docs/
|
| 47 |
docs
|
| 48 |
+
LOCAL_TESTING.md
|
main.py
CHANGED
|
@@ -65,8 +65,8 @@ def _draw_polyline(frame, lm, indices, w, h, color, thickness):
|
|
| 65 |
cv2.line(frame, _lm_px(lm, indices[i], w, h), _lm_px(lm, indices[i + 1], w, h), color, thickness, cv2.LINE_AA)
|
| 66 |
|
| 67 |
|
| 68 |
-
def _draw_face_mesh(frame, lm, w, h):
|
| 69 |
-
"""Draw tessellation, contours, eyebrows, nose, lips, eyes, irises, gaze lines."""
|
| 70 |
# Tessellation (gray triangular grid, semi-transparent)
|
| 71 |
overlay = frame.copy()
|
| 72 |
for s, e in _TESSELATION_CONNS:
|
|
@@ -92,7 +92,7 @@ def _draw_face_mesh(frame, lm, w, h):
|
|
| 92 |
for indices in [_LEFT_EAR_POINTS, _RIGHT_EAR_POINTS]:
|
| 93 |
for idx in indices:
|
| 94 |
cv2.circle(frame, _lm_px(lm, idx, w, h), 3, (0, 255, 255), -1, cv2.LINE_AA)
|
| 95 |
-
# Irises + gaze lines
|
| 96 |
for iris_idx, eye_inner, eye_outer in [
|
| 97 |
(FaceMeshDetector.LEFT_IRIS_INDICES, 133, 33),
|
| 98 |
(FaceMeshDetector.RIGHT_IRIS_INDICES, 362, 263),
|
|
@@ -104,10 +104,11 @@ def _draw_face_mesh(frame, lm, w, h):
|
|
| 104 |
radius = max(int(np.mean(radii)), 2)
|
| 105 |
cv2.circle(frame, tuple(center), radius, _MAGENTA, 2, cv2.LINE_AA)
|
| 106 |
cv2.circle(frame, tuple(center), 2, _WHITE, -1, cv2.LINE_AA)
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
|
|
|
| 111 |
|
| 112 |
|
| 113 |
def _draw_hud(frame, result, model_name):
|
|
@@ -356,8 +357,9 @@ class VideoTransformTrack(VideoStreamTrack):
|
|
| 356 |
# Draw face mesh + HUD on the video frame
|
| 357 |
h_f, w_f = img.shape[:2]
|
| 358 |
lm = out.get("landmarks")
|
|
|
|
| 359 |
if lm is not None:
|
| 360 |
-
_draw_face_mesh(img, lm, w_f, h_f)
|
| 361 |
_draw_hud(img, out, model_name)
|
| 362 |
else:
|
| 363 |
is_focused = False
|
|
@@ -959,6 +961,7 @@ async def websocket_endpoint(websocket: WebSocket):
|
|
| 959 |
"model": model_name,
|
| 960 |
"fc": frame_count,
|
| 961 |
"frame_count": frame_count,
|
|
|
|
| 962 |
}
|
| 963 |
if out is not None:
|
| 964 |
if out.get("yaw") is not None:
|
|
|
|
| 65 |
cv2.line(frame, _lm_px(lm, indices[i], w, h), _lm_px(lm, indices[i + 1], w, h), color, thickness, cv2.LINE_AA)
|
| 66 |
|
| 67 |
|
| 68 |
+
def _draw_face_mesh(frame, lm, w, h, draw_gaze_lines=False):
|
| 69 |
+
"""Draw tessellation, contours, eyebrows, nose, lips, eyes, irises, and optionally gaze lines."""
|
| 70 |
# Tessellation (gray triangular grid, semi-transparent)
|
| 71 |
overlay = frame.copy()
|
| 72 |
for s, e in _TESSELATION_CONNS:
|
|
|
|
| 92 |
for indices in [_LEFT_EAR_POINTS, _RIGHT_EAR_POINTS]:
|
| 93 |
for idx in indices:
|
| 94 |
cv2.circle(frame, _lm_px(lm, idx, w, h), 3, (0, 255, 255), -1, cv2.LINE_AA)
|
| 95 |
+
# Irises (always draw) + gaze lines (only when eye gaze is enabled)
|
| 96 |
for iris_idx, eye_inner, eye_outer in [
|
| 97 |
(FaceMeshDetector.LEFT_IRIS_INDICES, 133, 33),
|
| 98 |
(FaceMeshDetector.RIGHT_IRIS_INDICES, 362, 263),
|
|
|
|
| 104 |
radius = max(int(np.mean(radii)), 2)
|
| 105 |
cv2.circle(frame, tuple(center), radius, _MAGENTA, 2, cv2.LINE_AA)
|
| 106 |
cv2.circle(frame, tuple(center), 2, _WHITE, -1, cv2.LINE_AA)
|
| 107 |
+
if draw_gaze_lines:
|
| 108 |
+
eye_cx = int((lm[eye_inner, 0] + lm[eye_outer, 0]) / 2.0 * w)
|
| 109 |
+
eye_cy = int((lm[eye_inner, 1] + lm[eye_outer, 1]) / 2.0 * h)
|
| 110 |
+
dx, dy = center[0] - eye_cx, center[1] - eye_cy
|
| 111 |
+
cv2.line(frame, tuple(center), (int(center[0] + dx * 3), int(center[1] + dy * 3)), _RED, 1, cv2.LINE_AA)
|
| 112 |
|
| 113 |
|
| 114 |
def _draw_hud(frame, result, model_name):
|
|
|
|
| 357 |
# Draw face mesh + HUD on the video frame
|
| 358 |
h_f, w_f = img.shape[:2]
|
| 359 |
lm = out.get("landmarks")
|
| 360 |
+
eye_gaze_enabled = _l2cs_boost_enabled or model_name == "l2cs"
|
| 361 |
if lm is not None:
|
| 362 |
+
_draw_face_mesh(img, lm, w_f, h_f, draw_gaze_lines=eye_gaze_enabled)
|
| 363 |
_draw_hud(img, out, model_name)
|
| 364 |
else:
|
| 365 |
is_focused = False
|
|
|
|
| 961 |
"model": model_name,
|
| 962 |
"fc": frame_count,
|
| 963 |
"frame_count": frame_count,
|
| 964 |
+
"eye_gaze_enabled": _l2cs_boost_enabled or model_name == "l2cs",
|
| 965 |
}
|
| 966 |
if out is not None:
|
| 967 |
if out.get("yaw") is not None:
|
src/utils/VideoManagerLocal.js
CHANGED
|
@@ -163,7 +163,7 @@ export class VideoManagerLocal {
|
|
| 163 |
this.ws.onclose = null;
|
| 164 |
try {
|
| 165 |
this.ws.close();
|
| 166 |
-
} catch (_) {}
|
| 167 |
this.ws = null;
|
| 168 |
}
|
| 169 |
|
|
@@ -460,6 +460,7 @@ export class VideoManagerLocal {
|
|
| 460 |
on_screen: data.on_screen,
|
| 461 |
gaze_yaw: data.gaze_yaw,
|
| 462 |
gaze_pitch: data.gaze_pitch,
|
|
|
|
| 463 |
};
|
| 464 |
this.drawDetectionResult(detectionData);
|
| 465 |
break;
|
|
@@ -572,15 +573,15 @@ export class VideoManagerLocal {
|
|
| 572 |
152, 148, 176, 149, 150, 136, 172, 58, 132,
|
| 573 |
93, 234, 127, 162, 21, 54, 103, 67, 109, 10,
|
| 574 |
];
|
| 575 |
-
static LEFT_EYE = [33,7,163,144,145,153,154,155,133,173,157,158,159,160,161,246];
|
| 576 |
-
static RIGHT_EYE = [362,382,381,380,374,373,390,249,263,466,388,387,386,385,384,398];
|
| 577 |
-
static LEFT_IRIS = [468,469,470,471,472];
|
| 578 |
-
static RIGHT_IRIS = [473,474,475,476,477];
|
| 579 |
-
static LEFT_EYEBROW = [70,63,105,66,107,55,65,52,53,46];
|
| 580 |
-
static RIGHT_EYEBROW = [300,293,334,296,336,285,295,282,283,276];
|
| 581 |
-
static NOSE_BRIDGE = [6,197,195,5,4,1,19,94,2];
|
| 582 |
-
static LIPS_OUTER = [61,146,91,181,84,17,314,405,321,375,291,409,270,269,267,0,37,39,40,185,61];
|
| 583 |
-
static LIPS_INNER = [78,95,88,178,87,14,317,402,318,324,308,415,310,311,312,13,82,81,80,191,78];
|
| 584 |
static LEFT_EAR_POINTS = [33, 160, 158, 133, 153, 145];
|
| 585 |
static RIGHT_EAR_POINTS = [362, 385, 387, 263, 373, 380];
|
| 586 |
// Iris/eye corners for gaze lines
|
|
@@ -676,11 +677,12 @@ export class VideoManagerLocal {
|
|
| 676 |
outer: VideoManagerLocal.RIGHT_EYE_OUTER,
|
| 677 |
},
|
| 678 |
];
|
| 679 |
-
// Get L2CS gaze angles + on_screen state from latest detection data
|
| 680 |
const detection = this._lastDetection;
|
| 681 |
const gazeYaw = detection ? detection.gaze_yaw : undefined;
|
| 682 |
const gazePitch = detection ? detection.gaze_pitch : undefined;
|
| 683 |
const onScreen = detection ? detection.on_screen : undefined;
|
|
|
|
| 684 |
const hasL2CSGaze = gazeYaw !== undefined && gazePitch !== undefined;
|
| 685 |
const gazeLineColor = (onScreen === false) ? '#FF0000' : '#00FF00';
|
| 686 |
const gazeLineLength = 100;
|
|
@@ -712,57 +714,60 @@ export class VideoManagerLocal {
|
|
| 712 |
ctx.lineWidth = 1;
|
| 713 |
ctx.stroke();
|
| 714 |
|
| 715 |
-
// Gaze direction line —
|
| 716 |
-
if (
|
| 717 |
-
|
| 718 |
-
|
| 719 |
-
|
| 720 |
-
|
| 721 |
-
|
| 722 |
-
|
| 723 |
-
|
| 724 |
-
|
| 725 |
-
|
| 726 |
-
|
| 727 |
-
|
| 728 |
-
|
| 729 |
-
|
| 730 |
-
|
| 731 |
-
|
| 732 |
-
|
| 733 |
-
|
| 734 |
-
|
| 735 |
-
|
| 736 |
-
|
| 737 |
-
|
| 738 |
-
|
| 739 |
-
|
| 740 |
-
|
| 741 |
-
|
| 742 |
-
|
| 743 |
-
|
| 744 |
-
|
| 745 |
-
|
| 746 |
-
|
| 747 |
-
|
| 748 |
-
|
| 749 |
-
|
| 750 |
-
|
| 751 |
-
|
| 752 |
-
|
| 753 |
-
|
| 754 |
-
|
| 755 |
-
|
| 756 |
-
|
| 757 |
-
|
| 758 |
-
|
| 759 |
-
|
| 760 |
-
|
| 761 |
-
|
| 762 |
-
|
|
|
|
|
|
|
| 763 |
}
|
| 764 |
}
|
| 765 |
}
|
|
|
|
| 766 |
}
|
| 767 |
}
|
| 768 |
|
|
|
|
| 163 |
this.ws.onclose = null;
|
| 164 |
try {
|
| 165 |
this.ws.close();
|
| 166 |
+
} catch (_) { }
|
| 167 |
this.ws = null;
|
| 168 |
}
|
| 169 |
|
|
|
|
| 460 |
on_screen: data.on_screen,
|
| 461 |
gaze_yaw: data.gaze_yaw,
|
| 462 |
gaze_pitch: data.gaze_pitch,
|
| 463 |
+
eye_gaze_enabled: data.eye_gaze_enabled || false,
|
| 464 |
};
|
| 465 |
this.drawDetectionResult(detectionData);
|
| 466 |
break;
|
|
|
|
| 573 |
152, 148, 176, 149, 150, 136, 172, 58, 132,
|
| 574 |
93, 234, 127, 162, 21, 54, 103, 67, 109, 10,
|
| 575 |
];
|
| 576 |
+
static LEFT_EYE = [33, 7, 163, 144, 145, 153, 154, 155, 133, 173, 157, 158, 159, 160, 161, 246];
|
| 577 |
+
static RIGHT_EYE = [362, 382, 381, 380, 374, 373, 390, 249, 263, 466, 388, 387, 386, 385, 384, 398];
|
| 578 |
+
static LEFT_IRIS = [468, 469, 470, 471, 472];
|
| 579 |
+
static RIGHT_IRIS = [473, 474, 475, 476, 477];
|
| 580 |
+
static LEFT_EYEBROW = [70, 63, 105, 66, 107, 55, 65, 52, 53, 46];
|
| 581 |
+
static RIGHT_EYEBROW = [300, 293, 334, 296, 336, 285, 295, 282, 283, 276];
|
| 582 |
+
static NOSE_BRIDGE = [6, 197, 195, 5, 4, 1, 19, 94, 2];
|
| 583 |
+
static LIPS_OUTER = [61, 146, 91, 181, 84, 17, 314, 405, 321, 375, 291, 409, 270, 269, 267, 0, 37, 39, 40, 185, 61];
|
| 584 |
+
static LIPS_INNER = [78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 415, 310, 311, 312, 13, 82, 81, 80, 191, 78];
|
| 585 |
static LEFT_EAR_POINTS = [33, 160, 158, 133, 153, 145];
|
| 586 |
static RIGHT_EAR_POINTS = [362, 385, 387, 263, 373, 380];
|
| 587 |
// Iris/eye corners for gaze lines
|
|
|
|
| 677 |
outer: VideoManagerLocal.RIGHT_EYE_OUTER,
|
| 678 |
},
|
| 679 |
];
|
| 680 |
+
// Get L2CS gaze angles + on_screen state + eye gaze toggle from latest detection data
|
| 681 |
const detection = this._lastDetection;
|
| 682 |
const gazeYaw = detection ? detection.gaze_yaw : undefined;
|
| 683 |
const gazePitch = detection ? detection.gaze_pitch : undefined;
|
| 684 |
const onScreen = detection ? detection.on_screen : undefined;
|
| 685 |
+
const eyeGazeEnabled = detection ? detection.eye_gaze_enabled : false;
|
| 686 |
const hasL2CSGaze = gazeYaw !== undefined && gazePitch !== undefined;
|
| 687 |
const gazeLineColor = (onScreen === false) ? '#FF0000' : '#00FF00';
|
| 688 |
const gazeLineLength = 100;
|
|
|
|
| 714 |
ctx.lineWidth = 1;
|
| 715 |
ctx.stroke();
|
| 716 |
|
| 717 |
+
// Gaze direction line — only draw when eye gaze toggle is ON
|
| 718 |
+
if (eyeGazeEnabled) {
|
| 719 |
+
if (hasL2CSGaze) {
|
| 720 |
+
// L2CS pitch/yaw in radians -> pixel direction vector
|
| 721 |
+
// Matches upstream L2CS-Net vis.py draw_gaze formula:
|
| 722 |
+
// dx = -length * sin(pitch) * cos(yaw)
|
| 723 |
+
// dy = -length * sin(yaw)
|
| 724 |
+
const dx = -gazeLineLength * Math.sin(gazePitch) * Math.cos(gazeYaw);
|
| 725 |
+
const dy = -gazeLineLength * Math.sin(gazeYaw);
|
| 726 |
+
const ex = cx + dx;
|
| 727 |
+
const ey = cy + dy;
|
| 728 |
+
|
| 729 |
+
// Main gaze line (thick, color-coded)
|
| 730 |
+
ctx.beginPath();
|
| 731 |
+
ctx.moveTo(cx, cy);
|
| 732 |
+
ctx.lineTo(ex, ey);
|
| 733 |
+
ctx.strokeStyle = gazeLineColor;
|
| 734 |
+
ctx.lineWidth = 3;
|
| 735 |
+
ctx.stroke();
|
| 736 |
+
|
| 737 |
+
// Arrowhead
|
| 738 |
+
const angle = Math.atan2(ey - cy, ex - cx);
|
| 739 |
+
const arrowLen = 10;
|
| 740 |
+
ctx.beginPath();
|
| 741 |
+
ctx.moveTo(ex, ey);
|
| 742 |
+
ctx.lineTo(ex - arrowLen * Math.cos(angle - 0.4), ey - arrowLen * Math.sin(angle - 0.4));
|
| 743 |
+
ctx.moveTo(ex, ey);
|
| 744 |
+
ctx.lineTo(ex - arrowLen * Math.cos(angle + 0.4), ey - arrowLen * Math.sin(angle + 0.4));
|
| 745 |
+
ctx.strokeStyle = gazeLineColor;
|
| 746 |
+
ctx.lineWidth = 2;
|
| 747 |
+
ctx.stroke();
|
| 748 |
+
} else {
|
| 749 |
+
// Geometric fallback: iris displacement from eye center (scaled up)
|
| 750 |
+
const innerPt = _get(inner);
|
| 751 |
+
const outerPt = _get(outer);
|
| 752 |
+
if (innerPt && outerPt) {
|
| 753 |
+
const eyeCx = (innerPt[0] + outerPt[0]) / 2.0 * w;
|
| 754 |
+
const eyeCy = (innerPt[1] + outerPt[1]) / 2.0 * h;
|
| 755 |
+
const fdx = cx - eyeCx;
|
| 756 |
+
const fdy = cy - eyeCy;
|
| 757 |
+
const flen = Math.hypot(fdx, fdy);
|
| 758 |
+
if (flen > 0.5) {
|
| 759 |
+
const scale = gazeLineLength / flen;
|
| 760 |
+
ctx.beginPath();
|
| 761 |
+
ctx.moveTo(cx, cy);
|
| 762 |
+
ctx.lineTo(cx + fdx * scale, cy + fdy * scale);
|
| 763 |
+
ctx.strokeStyle = '#00FFFF';
|
| 764 |
+
ctx.lineWidth = 2;
|
| 765 |
+
ctx.stroke();
|
| 766 |
+
}
|
| 767 |
}
|
| 768 |
}
|
| 769 |
}
|
| 770 |
+
// When eye gaze is OFF, no gaze lines are drawn
|
| 771 |
}
|
| 772 |
}
|
| 773 |
|