k23099462 commited on
Commit
3dc18ac
·
1 Parent(s): 0d448a6

fix: eye gaze toggle now properly controls gaze line visibility

Browse files
Files changed (3) hide show
  1. .gitignore +1 -0
  2. main.py +11 -8
  3. src/utils/VideoManagerLocal.js +64 -59
.gitignore CHANGED
@@ -45,3 +45,4 @@ test_focus_guard.db
45
  __pycache__/
46
  docs/
47
  docs
 
 
45
  __pycache__/
46
  docs/
47
  docs
48
+ LOCAL_TESTING.md
main.py CHANGED
@@ -65,8 +65,8 @@ def _draw_polyline(frame, lm, indices, w, h, color, thickness):
65
  cv2.line(frame, _lm_px(lm, indices[i], w, h), _lm_px(lm, indices[i + 1], w, h), color, thickness, cv2.LINE_AA)
66
 
67
 
68
- def _draw_face_mesh(frame, lm, w, h):
69
- """Draw tessellation, contours, eyebrows, nose, lips, eyes, irises, gaze lines."""
70
  # Tessellation (gray triangular grid, semi-transparent)
71
  overlay = frame.copy()
72
  for s, e in _TESSELATION_CONNS:
@@ -92,7 +92,7 @@ def _draw_face_mesh(frame, lm, w, h):
92
  for indices in [_LEFT_EAR_POINTS, _RIGHT_EAR_POINTS]:
93
  for idx in indices:
94
  cv2.circle(frame, _lm_px(lm, idx, w, h), 3, (0, 255, 255), -1, cv2.LINE_AA)
95
- # Irises + gaze lines
96
  for iris_idx, eye_inner, eye_outer in [
97
  (FaceMeshDetector.LEFT_IRIS_INDICES, 133, 33),
98
  (FaceMeshDetector.RIGHT_IRIS_INDICES, 362, 263),
@@ -104,10 +104,11 @@ def _draw_face_mesh(frame, lm, w, h):
104
  radius = max(int(np.mean(radii)), 2)
105
  cv2.circle(frame, tuple(center), radius, _MAGENTA, 2, cv2.LINE_AA)
106
  cv2.circle(frame, tuple(center), 2, _WHITE, -1, cv2.LINE_AA)
107
- eye_cx = int((lm[eye_inner, 0] + lm[eye_outer, 0]) / 2.0 * w)
108
- eye_cy = int((lm[eye_inner, 1] + lm[eye_outer, 1]) / 2.0 * h)
109
- dx, dy = center[0] - eye_cx, center[1] - eye_cy
110
- cv2.line(frame, tuple(center), (int(center[0] + dx * 3), int(center[1] + dy * 3)), _RED, 1, cv2.LINE_AA)
 
111
 
112
 
113
  def _draw_hud(frame, result, model_name):
@@ -356,8 +357,9 @@ class VideoTransformTrack(VideoStreamTrack):
356
  # Draw face mesh + HUD on the video frame
357
  h_f, w_f = img.shape[:2]
358
  lm = out.get("landmarks")
 
359
  if lm is not None:
360
- _draw_face_mesh(img, lm, w_f, h_f)
361
  _draw_hud(img, out, model_name)
362
  else:
363
  is_focused = False
@@ -959,6 +961,7 @@ async def websocket_endpoint(websocket: WebSocket):
959
  "model": model_name,
960
  "fc": frame_count,
961
  "frame_count": frame_count,
 
962
  }
963
  if out is not None:
964
  if out.get("yaw") is not None:
 
65
  cv2.line(frame, _lm_px(lm, indices[i], w, h), _lm_px(lm, indices[i + 1], w, h), color, thickness, cv2.LINE_AA)
66
 
67
 
68
+ def _draw_face_mesh(frame, lm, w, h, draw_gaze_lines=False):
69
+ """Draw tessellation, contours, eyebrows, nose, lips, eyes, irises, and optionally gaze lines."""
70
  # Tessellation (gray triangular grid, semi-transparent)
71
  overlay = frame.copy()
72
  for s, e in _TESSELATION_CONNS:
 
92
  for indices in [_LEFT_EAR_POINTS, _RIGHT_EAR_POINTS]:
93
  for idx in indices:
94
  cv2.circle(frame, _lm_px(lm, idx, w, h), 3, (0, 255, 255), -1, cv2.LINE_AA)
95
+ # Irises (always draw) + gaze lines (only when eye gaze is enabled)
96
  for iris_idx, eye_inner, eye_outer in [
97
  (FaceMeshDetector.LEFT_IRIS_INDICES, 133, 33),
98
  (FaceMeshDetector.RIGHT_IRIS_INDICES, 362, 263),
 
104
  radius = max(int(np.mean(radii)), 2)
105
  cv2.circle(frame, tuple(center), radius, _MAGENTA, 2, cv2.LINE_AA)
106
  cv2.circle(frame, tuple(center), 2, _WHITE, -1, cv2.LINE_AA)
107
+ if draw_gaze_lines:
108
+ eye_cx = int((lm[eye_inner, 0] + lm[eye_outer, 0]) / 2.0 * w)
109
+ eye_cy = int((lm[eye_inner, 1] + lm[eye_outer, 1]) / 2.0 * h)
110
+ dx, dy = center[0] - eye_cx, center[1] - eye_cy
111
+ cv2.line(frame, tuple(center), (int(center[0] + dx * 3), int(center[1] + dy * 3)), _RED, 1, cv2.LINE_AA)
112
 
113
 
114
  def _draw_hud(frame, result, model_name):
 
357
  # Draw face mesh + HUD on the video frame
358
  h_f, w_f = img.shape[:2]
359
  lm = out.get("landmarks")
360
+ eye_gaze_enabled = _l2cs_boost_enabled or model_name == "l2cs"
361
  if lm is not None:
362
+ _draw_face_mesh(img, lm, w_f, h_f, draw_gaze_lines=eye_gaze_enabled)
363
  _draw_hud(img, out, model_name)
364
  else:
365
  is_focused = False
 
961
  "model": model_name,
962
  "fc": frame_count,
963
  "frame_count": frame_count,
964
+ "eye_gaze_enabled": _l2cs_boost_enabled or model_name == "l2cs",
965
  }
966
  if out is not None:
967
  if out.get("yaw") is not None:
src/utils/VideoManagerLocal.js CHANGED
@@ -163,7 +163,7 @@ export class VideoManagerLocal {
163
  this.ws.onclose = null;
164
  try {
165
  this.ws.close();
166
- } catch (_) {}
167
  this.ws = null;
168
  }
169
 
@@ -460,6 +460,7 @@ export class VideoManagerLocal {
460
  on_screen: data.on_screen,
461
  gaze_yaw: data.gaze_yaw,
462
  gaze_pitch: data.gaze_pitch,
 
463
  };
464
  this.drawDetectionResult(detectionData);
465
  break;
@@ -572,15 +573,15 @@ export class VideoManagerLocal {
572
  152, 148, 176, 149, 150, 136, 172, 58, 132,
573
  93, 234, 127, 162, 21, 54, 103, 67, 109, 10,
574
  ];
575
- static LEFT_EYE = [33,7,163,144,145,153,154,155,133,173,157,158,159,160,161,246];
576
- static RIGHT_EYE = [362,382,381,380,374,373,390,249,263,466,388,387,386,385,384,398];
577
- static LEFT_IRIS = [468,469,470,471,472];
578
- static RIGHT_IRIS = [473,474,475,476,477];
579
- static LEFT_EYEBROW = [70,63,105,66,107,55,65,52,53,46];
580
- static RIGHT_EYEBROW = [300,293,334,296,336,285,295,282,283,276];
581
- static NOSE_BRIDGE = [6,197,195,5,4,1,19,94,2];
582
- static LIPS_OUTER = [61,146,91,181,84,17,314,405,321,375,291,409,270,269,267,0,37,39,40,185,61];
583
- static LIPS_INNER = [78,95,88,178,87,14,317,402,318,324,308,415,310,311,312,13,82,81,80,191,78];
584
  static LEFT_EAR_POINTS = [33, 160, 158, 133, 153, 145];
585
  static RIGHT_EAR_POINTS = [362, 385, 387, 263, 373, 380];
586
  // Iris/eye corners for gaze lines
@@ -676,11 +677,12 @@ export class VideoManagerLocal {
676
  outer: VideoManagerLocal.RIGHT_EYE_OUTER,
677
  },
678
  ];
679
- // Get L2CS gaze angles + on_screen state from latest detection data
680
  const detection = this._lastDetection;
681
  const gazeYaw = detection ? detection.gaze_yaw : undefined;
682
  const gazePitch = detection ? detection.gaze_pitch : undefined;
683
  const onScreen = detection ? detection.on_screen : undefined;
 
684
  const hasL2CSGaze = gazeYaw !== undefined && gazePitch !== undefined;
685
  const gazeLineColor = (onScreen === false) ? '#FF0000' : '#00FF00';
686
  const gazeLineLength = 100;
@@ -712,57 +714,60 @@ export class VideoManagerLocal {
712
  ctx.lineWidth = 1;
713
  ctx.stroke();
714
 
715
- // Gaze direction line — use L2CS angles when available, else geometric fallback
716
- if (hasL2CSGaze) {
717
- // L2CS pitch/yaw in radians -> pixel direction vector
718
- // Matches upstream L2CS-Net vis.py draw_gaze formula:
719
- // dx = -length * sin(pitch) * cos(yaw)
720
- // dy = -length * sin(yaw)
721
- const dx = -gazeLineLength * Math.sin(gazePitch) * Math.cos(gazeYaw);
722
- const dy = -gazeLineLength * Math.sin(gazeYaw);
723
- const ex = cx + dx;
724
- const ey = cy + dy;
725
-
726
- // Main gaze line (thick, color-coded)
727
- ctx.beginPath();
728
- ctx.moveTo(cx, cy);
729
- ctx.lineTo(ex, ey);
730
- ctx.strokeStyle = gazeLineColor;
731
- ctx.lineWidth = 3;
732
- ctx.stroke();
733
-
734
- // Arrowhead
735
- const angle = Math.atan2(ey - cy, ex - cx);
736
- const arrowLen = 10;
737
- ctx.beginPath();
738
- ctx.moveTo(ex, ey);
739
- ctx.lineTo(ex - arrowLen * Math.cos(angle - 0.4), ey - arrowLen * Math.sin(angle - 0.4));
740
- ctx.moveTo(ex, ey);
741
- ctx.lineTo(ex - arrowLen * Math.cos(angle + 0.4), ey - arrowLen * Math.sin(angle + 0.4));
742
- ctx.strokeStyle = gazeLineColor;
743
- ctx.lineWidth = 2;
744
- ctx.stroke();
745
- } else {
746
- // Geometric fallback: iris displacement from eye center (scaled up)
747
- const innerPt = _get(inner);
748
- const outerPt = _get(outer);
749
- if (innerPt && outerPt) {
750
- const eyeCx = (innerPt[0] + outerPt[0]) / 2.0 * w;
751
- const eyeCy = (innerPt[1] + outerPt[1]) / 2.0 * h;
752
- const fdx = cx - eyeCx;
753
- const fdy = cy - eyeCy;
754
- const flen = Math.hypot(fdx, fdy);
755
- if (flen > 0.5) {
756
- const scale = gazeLineLength / flen;
757
- ctx.beginPath();
758
- ctx.moveTo(cx, cy);
759
- ctx.lineTo(cx + fdx * scale, cy + fdy * scale);
760
- ctx.strokeStyle = '#00FFFF';
761
- ctx.lineWidth = 2;
762
- ctx.stroke();
 
 
763
  }
764
  }
765
  }
 
766
  }
767
  }
768
 
 
163
  this.ws.onclose = null;
164
  try {
165
  this.ws.close();
166
+ } catch (_) { }
167
  this.ws = null;
168
  }
169
 
 
460
  on_screen: data.on_screen,
461
  gaze_yaw: data.gaze_yaw,
462
  gaze_pitch: data.gaze_pitch,
463
+ eye_gaze_enabled: data.eye_gaze_enabled || false,
464
  };
465
  this.drawDetectionResult(detectionData);
466
  break;
 
573
  152, 148, 176, 149, 150, 136, 172, 58, 132,
574
  93, 234, 127, 162, 21, 54, 103, 67, 109, 10,
575
  ];
576
+ static LEFT_EYE = [33, 7, 163, 144, 145, 153, 154, 155, 133, 173, 157, 158, 159, 160, 161, 246];
577
+ static RIGHT_EYE = [362, 382, 381, 380, 374, 373, 390, 249, 263, 466, 388, 387, 386, 385, 384, 398];
578
+ static LEFT_IRIS = [468, 469, 470, 471, 472];
579
+ static RIGHT_IRIS = [473, 474, 475, 476, 477];
580
+ static LEFT_EYEBROW = [70, 63, 105, 66, 107, 55, 65, 52, 53, 46];
581
+ static RIGHT_EYEBROW = [300, 293, 334, 296, 336, 285, 295, 282, 283, 276];
582
+ static NOSE_BRIDGE = [6, 197, 195, 5, 4, 1, 19, 94, 2];
583
+ static LIPS_OUTER = [61, 146, 91, 181, 84, 17, 314, 405, 321, 375, 291, 409, 270, 269, 267, 0, 37, 39, 40, 185, 61];
584
+ static LIPS_INNER = [78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 415, 310, 311, 312, 13, 82, 81, 80, 191, 78];
585
  static LEFT_EAR_POINTS = [33, 160, 158, 133, 153, 145];
586
  static RIGHT_EAR_POINTS = [362, 385, 387, 263, 373, 380];
587
  // Iris/eye corners for gaze lines
 
677
  outer: VideoManagerLocal.RIGHT_EYE_OUTER,
678
  },
679
  ];
680
+ // Get L2CS gaze angles + on_screen state + eye gaze toggle from latest detection data
681
  const detection = this._lastDetection;
682
  const gazeYaw = detection ? detection.gaze_yaw : undefined;
683
  const gazePitch = detection ? detection.gaze_pitch : undefined;
684
  const onScreen = detection ? detection.on_screen : undefined;
685
+ const eyeGazeEnabled = detection ? detection.eye_gaze_enabled : false;
686
  const hasL2CSGaze = gazeYaw !== undefined && gazePitch !== undefined;
687
  const gazeLineColor = (onScreen === false) ? '#FF0000' : '#00FF00';
688
  const gazeLineLength = 100;
 
714
  ctx.lineWidth = 1;
715
  ctx.stroke();
716
 
717
+ // Gaze direction line — only draw when eye gaze toggle is ON
718
+ if (eyeGazeEnabled) {
719
+ if (hasL2CSGaze) {
720
+ // L2CS pitch/yaw in radians -> pixel direction vector
721
+ // Matches upstream L2CS-Net vis.py draw_gaze formula:
722
+ // dx = -length * sin(pitch) * cos(yaw)
723
+ // dy = -length * sin(yaw)
724
+ const dx = -gazeLineLength * Math.sin(gazePitch) * Math.cos(gazeYaw);
725
+ const dy = -gazeLineLength * Math.sin(gazeYaw);
726
+ const ex = cx + dx;
727
+ const ey = cy + dy;
728
+
729
+ // Main gaze line (thick, color-coded)
730
+ ctx.beginPath();
731
+ ctx.moveTo(cx, cy);
732
+ ctx.lineTo(ex, ey);
733
+ ctx.strokeStyle = gazeLineColor;
734
+ ctx.lineWidth = 3;
735
+ ctx.stroke();
736
+
737
+ // Arrowhead
738
+ const angle = Math.atan2(ey - cy, ex - cx);
739
+ const arrowLen = 10;
740
+ ctx.beginPath();
741
+ ctx.moveTo(ex, ey);
742
+ ctx.lineTo(ex - arrowLen * Math.cos(angle - 0.4), ey - arrowLen * Math.sin(angle - 0.4));
743
+ ctx.moveTo(ex, ey);
744
+ ctx.lineTo(ex - arrowLen * Math.cos(angle + 0.4), ey - arrowLen * Math.sin(angle + 0.4));
745
+ ctx.strokeStyle = gazeLineColor;
746
+ ctx.lineWidth = 2;
747
+ ctx.stroke();
748
+ } else {
749
+ // Geometric fallback: iris displacement from eye center (scaled up)
750
+ const innerPt = _get(inner);
751
+ const outerPt = _get(outer);
752
+ if (innerPt && outerPt) {
753
+ const eyeCx = (innerPt[0] + outerPt[0]) / 2.0 * w;
754
+ const eyeCy = (innerPt[1] + outerPt[1]) / 2.0 * h;
755
+ const fdx = cx - eyeCx;
756
+ const fdy = cy - eyeCy;
757
+ const flen = Math.hypot(fdx, fdy);
758
+ if (flen > 0.5) {
759
+ const scale = gazeLineLength / flen;
760
+ ctx.beginPath();
761
+ ctx.moveTo(cx, cy);
762
+ ctx.lineTo(cx + fdx * scale, cy + fdy * scale);
763
+ ctx.strokeStyle = '#00FFFF';
764
+ ctx.lineWidth = 2;
765
+ ctx.stroke();
766
+ }
767
  }
768
  }
769
  }
770
+ // When eye gaze is OFF, no gaze lines are drawn
771
  }
772
  }
773