Michael Rabinovich Cursor commited on
Commit
987f284
·
1 Parent(s): ec67f32

Submit gate: accept unparseable candidates (eval scores them invalid/0)

Browse files

A present-but-unloadable output.step (empty, or parses to no geometry) was
rejecting the entire submission. The evaluator's CAD-validity gate already
scores any non-valid solid as cad_score=0 (status=invalid), so a bad candidate
is a legitimate zero-scoring outcome, not a submission error. The gate now logs
and lets it through instead of blocking the whole zip.

Co-authored-by: Cursor <cursoragent@cursor.com>

Files changed (1) hide show
  1. submit.py +19 -9
submit.py CHANGED
@@ -716,22 +716,32 @@ def _validate_candidates_parseable(unpacked: Path, fixture_names: set[str]) -> N
716
  # Missing output is a valid benchmark outcome: the evaluator writes
717
  # status="missing" and the fixture contributes cad_score=0.
718
  return
 
 
 
 
 
 
719
  if candidate.stat().st_size == 0:
720
- raise _ValidationError(
721
- f"Sample `{name}` has an empty `{candidate.name}`."
 
722
  )
 
723
  suffix = candidate.suffix.lower()
724
  if suffix in {".step", ".stp"}:
725
  try:
726
  parse_step(candidate)
727
- except RuntimeError as e:
728
- raise _ValidationError(
729
- f"Sample `{name}` has an `{candidate.name}` that is not "
730
- f"loadable as STEP geometry: {e}"
731
- ) from e
 
732
  else: # pragma: no cover - _candidate_path constrains suffixes
733
- raise _ValidationError(
734
- f"Sample `{name}` uses unsupported candidate file `{candidate.name}`."
 
735
  )
736
 
737
  with ThreadPoolExecutor(
 
716
  # Missing output is a valid benchmark outcome: the evaluator writes
717
  # status="missing" and the fixture contributes cad_score=0.
718
  return
719
+ # A present-but-unloadable candidate is NOT a submission error. The
720
+ # evaluator's CAD-validity gate scores any non-valid solid as
721
+ # cad_score=0 (status="invalid"), so an empty or unparseable file is a
722
+ # legitimate (zero-scoring) outcome, not a reason to reject the whole
723
+ # submission. We only log here; the gate never blocks on candidate
724
+ # content, so one bad part can't sink an otherwise-valid submission.
725
  if candidate.stat().st_size == 0:
726
+ logger.warning(
727
+ "Sample %s has an empty %s; will score 0 (invalid).",
728
+ name, candidate.name,
729
  )
730
+ return
731
  suffix = candidate.suffix.lower()
732
  if suffix in {".step", ".stp"}:
733
  try:
734
  parse_step(candidate)
735
+ except Exception as e: # noqa: BLE001 - non-fatal; eval scores it 0
736
+ logger.warning(
737
+ "Sample %s has an %s that is not loadable as STEP (%s); "
738
+ "will score 0 (invalid).",
739
+ name, candidate.name, e,
740
+ )
741
  else: # pragma: no cover - _candidate_path constrains suffixes
742
+ logger.warning(
743
+ "Sample %s uses unexpected candidate file %s; will score 0.",
744
+ name, candidate.name,
745
  )
746
 
747
  with ThreadPoolExecutor(