server/gate.py

# SPDX-License-Identifier: BSD-3-Clause

"""The admission check a submission has to clear before anyone judges it.

Nothing here is a matter of taste. The gate answers one narrow question: is
this a sketch that used the watercolour library to put paint on a canvas? A
submission that fails scores zero and is never sent to the judge, which is what
keeps a run affordable while a model is still producing garbage.

Two of the checks are about honesty rather than competence. `bare_primitives`
catches painting with plain p5 shapes, which yields a picture that can score on
composition while dodging the medium the task is about. `external_access`
catches loading somebody else's painting, which is the only way to score well
without painting at all.

The paint floor is calibrated against measured renders: real sketches from a 4B
model covered 2.6% to 5.0% of the canvas, a sketch that errored before painting
covered 0.07%, and an empty canvas covers 0%. The floor sits between them.
"""

from __future__ import annotations

from dataclasses import dataclass, field
from typing import Any

try:
    from .render import Render, RenderError, SketchRenderer
    from .sketch_source import extract_sketch, inspect_source, SourceError, SourceReport
except ImportError:
    from server.render import Render, RenderError, SketchRenderer
    from server.sketch_source import (
        extract_sketch,
        inspect_source,
        SourceError,
        SourceReport,
    )

MIN_PAINT_FRACTION = 0.005

# Painting calls a sketch must make to count as having used the library. One is
# enough: a single wash is a legitimate, if minimal, watercolour.
MIN_PAINTING_CALLS = 1

# Two attempts at the render. See `GateResult.render_unavailable` for the measurement
# that says a failure here is the browser rather than the sketch.
RENDER_ATTEMPTS = 2


@dataclass(frozen=True)
class GateResult:
    """Outcome of the admission check.

    Attributes:
        passed (`bool`):
            Whether the submission cleared every check.
        violations (`list[str]`):
            Reasons for rejection, empty when `passed` is `True`.
        source (`SourceReport` or `None`):
            The source inventory, `None` if no sketch could be extracted.
        render ([`Render`] or `None`):
            The render, `None` if the source was rejected before rendering or
            the sketch produced no canvas.
        render_unavailable (`bool`):
            The browser never produced a canvas and never reported an error.
            That is not the same as a bad sketch, and scoring it zero trains the
            policy away from code that was probably fine: of every `render_failed`
            recorded across eight runs, not one carried a JavaScript error, and all
            of them had already cleared the static checks for entry points and
            WEBGL. 37 of 2432 rollouts (1.5%) took a phantom zero this way, and in
            one run it reached 5.2% of rollouts and 35% of steps.
    """

    passed: bool
    violations: list[str] = field(default_factory=list)
    source: SourceReport | None = None
    render: Render | None = None
    render_unavailable: bool = False

    @property
    def png(self) -> bytes | None:
        """`bytes` or `None`: The painting, when one was produced."""
        return self.render.png if self.render is not None else None

    def to_dict(self) -> dict[str, Any]:
        """Return a JSON-serialisable view of the verdict."""
        return {
            "passed": self.passed,
            "violations": list(self.violations),
            "source": self.source.to_dict() if self.source else None,
            "render": self.render.to_dict() if self.render else None,
            "render_unavailable": self.render_unavailable,
        }


def _source_violations(report: SourceReport) -> list[str]:
    """Return the reasons a source is not worth rendering."""
    violations = []
    if not report.balanced:
        violations.append("truncated")
    # Only `setup` is required. Requiring `draw` too was rejecting paintings
    # that paint perfectly well: a sketch that does all its work in `setup`
    # renders the same picture as the equivalent one split across both, measured
    # at the same paint coverage. Demanding the split flattened the reward for no
    # reason a picture could show.
    if not report.has_setup:
        violations.append("missing_entry_points")
    if not report.webgl:
        violations.append("not_webgl")
    if report.external_access:
        violations.append("external_access")
    if report.writes_text:
        violations.append("text_label")
    if report.bare_primitives:
        violations.append("bare_primitives")
    # A method that is not on `brush` at all throws inside `draw`, the render
    # stops, and what comes back is whatever had been painted first: usually
    # nothing, reported as `blank_canvas`. That misnames the cause, and it cost a
    # day of blaming off-canvas coordinates. Measured on a twelve-sample probe of
    # a 35B, the four lowest-coverage submissions were exactly the four that threw
    # (`brush.lineWidth`, `brush.rotate`, and two undefined variables), and two of
    # them still cleared the gate at 0.7% coverage and were paid for a crash.
    #
    # `unknown_calls` was already computed and never read. It only matches names
    # called on `brush`, so helper functions, `Math.*` and p5 globals are
    # untouched: measured over the 56 reference sources, none is flagged. The
    # twenty-eight real p5.brush methods the prompt does not list are in
    # `KNOWN_CALLS` and still pass, so this rejects only what cannot exist.
    if report.unknown_calls:
        violations.append("unknown_brush_method")
    if len(report.painting_calls) < MIN_PAINTING_CALLS:
        violations.append("no_painting_calls")
    return violations


async def run_gate(response: str, renderer: SketchRenderer) -> GateResult:
    """Check a submission and, if it is worth it, render it.

    Args:
        response (`str`):
            The model's raw reply.
        renderer ([`SketchRenderer`]):
            The browser to render with.

    Returns:
        [`GateResult`]: The verdict, carrying the render so the judge does not
            pay to produce it again.

    Examples:

    ```python
    result = await run_gate(reply, renderer)
    if result.passed:
        print(result.render.paint_fraction)
    ```
    """
    try:
        source = extract_sketch(response)
    except SourceError:
        return GateResult(passed=False, violations=["no_sketch_in_response"])

    report = inspect_source(source)
    violations = _source_violations(report)
    if violations:
        return GateResult(passed=False, violations=violations, source=report)

    # Retried for the reason given on `render_unavailable`, and the retry is what
    # separates the two cases: a sketch that genuinely draws nothing fails every
    # time, while a browser that dropped the page usually succeeds on the next one.
    for intento in range(RENDER_ATTEMPTS):
        try:
            render = await renderer.render(source)
            break
        except RenderError:
            print(
                f"render attempt {intento + 1}/{RENDER_ATTEMPTS} produced no canvas",
                flush=True,
            )
            if intento == RENDER_ATTEMPTS - 1:
                return GateResult(
                    passed=False,
                    violations=["render_failed"],
                    source=report,
                    render_unavailable=True,
                )

    if render.paint_fraction < MIN_PAINT_FRACTION:
        return GateResult(
            passed=False,
            violations=["blank_canvas"],
            source=report,
            render=render,
        )
    return GateResult(passed=True, source=report, render=render)