Spaces:
Running
Running
File size: 2,497 Bytes
72bc633 44b0b12 72bc633 44b0b12 72bc633 44b0b12 72bc633 44b0b12 72bc633 44b0b12 72bc633 44b0b12 72bc633 58f6308 72bc633 44b0b12 72bc633 44b0b12 72bc633 44b0b12 72bc633 58f6308 44b0b12 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | """
PatchHawk task graders — referenced in openenv.yaml.
Each grader receives the environment and a trajectory (list of
(action, observation) tuples), and returns a float score strictly in (0, 1).
"""
from __future__ import annotations
from typing import TYPE_CHECKING, List, Tuple
if TYPE_CHECKING:
from patchhawk.agent.environment import PatchHawkEnv
from patchhawk.env_models import PatchHawkAction, PatchHawkObservation
# Action constants (mirrored from PatchHawkEnv)
_ANALYZE = 0
_EXECUTE_SANDBOX = 1
_BLOCK_PR = 2
_SUBMIT_PATCH = 3
_REQUEST_REVIEW = 4
def grade_easy(
env: "PatchHawkEnv",
trajectory: List[Tuple["PatchHawkAction", "PatchHawkObservation"]],
) -> float:
"""Easy task grader — typosquatting detection.
Returns 0.99 if:
- The last action is BLOCK_PR or SUBMIT_PATCH
- The scenario's attack_type is "typosquatting"
"""
if not trajectory:
return 0.01
last_action, _last_obs = trajectory[-1]
scenario = env.current_scenario or {}
if last_action.action_type in (_BLOCK_PR, _SUBMIT_PATCH):
if scenario.get("attack_type") == "typosquatting":
return 0.99
return 0.01
def grade_medium(
env: "PatchHawkEnv",
trajectory: List[Tuple["PatchHawkAction", "PatchHawkObservation"]],
) -> float:
"""Medium task grader — obfuscated exec detection.
Returns 0.99 if:
- EXECUTE_SANDBOX was used at least once in the trajectory
- The last action is BLOCK_PR or SUBMIT_PATCH
- The scenario's attack_type is "obfuscated_exec"
"""
if not trajectory:
return 0.01
used_sandbox = any(a.action_type == _EXECUTE_SANDBOX for a, _o in trajectory)
last_action, _last_obs = trajectory[-1]
scenario = env.current_scenario or {}
if (
used_sandbox
and last_action.action_type in (_BLOCK_PR, _SUBMIT_PATCH)
and scenario.get("attack_type") == "obfuscated_exec"
):
return 0.99
return 0.01
def grade_hard(
env: "PatchHawkEnv",
trajectory: List[Tuple["PatchHawkAction", "PatchHawkObservation"]],
) -> float:
"""Hard task grader — validated patch submission.
Returns 0.99 if:
- The last action is SUBMIT_PATCH
- env.state.patch_validated is True
"""
if not trajectory:
return 0.01
last_action, _last_obs = trajectory[-1]
if last_action.action_type == _SUBMIT_PATCH and env.state.patch_validated:
return 0.99
return 0.01
|