tests/test_agent_system.py

import os
import unittest
from unittest.mock import patch

from agent_system import (
    AgentSettings,
    LocalAgentSystem,
    ValidationDecision,
    clean_submission_value,
    is_placeholder_answer,
    question_transform_hints,
)
from model_config import DEFAULT_OLLAMA_MODEL


class AgentSettingsTests(unittest.TestCase):
    def test_gemma_4_is_the_shared_default(self) -> None:
        with patch.dict(os.environ, {}, clear=True):
            settings = AgentSettings.from_env()
        self.assertEqual(settings.text_model, DEFAULT_OLLAMA_MODEL)
        self.assertEqual(settings.multimodal_model, DEFAULT_OLLAMA_MODEL)
        self.assertEqual(settings.max_research_steps, 6)
        self.assertEqual(settings.max_validation_retries, 2)


class ValidationDecisionTests(unittest.TestCase):
    def test_pass_requires_consistent_supported_evidence(self) -> None:
        supported = ValidationDecision.from_payload(
            {
                "status": "pass",
                "answer": "42",
                "supporting_evidence": ["The report directly establishes 42."],
                "issues": [],
                "required_research": [],
                "rerun_plan": False,
            }
        )
        inconsistent = ValidationDecision.from_payload(
            {
                "status": "pass",
                "answer": "42",
                "supporting_evidence": ["One source says 42."],
                "issues": ["Another source says 43."],
                "required_research": [],
                "rerun_plan": False,
            }
        )
        self.assertTrue(supported.passed)
        self.assertFalse(inconsistent.passed)


class _FakeStructuredAgent:
    def __init__(self, responses):
        self.responses = iter(responses)

    def run(self, prompt, schema):
        return next(self.responses)


class _FakeResearcher:
    def __init__(self, responses):
        self.responses = iter(responses)
        self.calls = 0

    def run(self, prompt, reset):
        self.calls += 1
        return next(self.responses)


class RetryLoopTests(unittest.TestCase):
    def test_validation_feedback_triggers_research_retry(self) -> None:
        system = LocalAgentSystem.__new__(LocalAgentSystem)
        system.settings = AgentSettings(
            ollama_base_url="http://localhost:11434",
            text_model=DEFAULT_OLLAMA_MODEL,
            multimodal_model=DEFAULT_OLLAMA_MODEL,
            context_size=8192,
            max_research_steps=4,
            max_validation_retries=2,
        )
        plan = {
            "answer_format": "integer",
            "facts_to_verify": ["the exact count"],
            "research_queries": ["authoritative count"],
            "calculations": [],
            "attachment_use": "none",
        }
        system.planner = _FakeStructuredAgent([plan])
        system.researcher = _FakeResearcher(
            ["Conflicting evidence: 41 or 42", "Two sources establish 42"]
        )
        system.validator = _FakeStructuredAgent(
            [
                {
                    "status": "retry",
                    "answer": "",
                    "supporting_evidence": [],
                    "issues": ["The count is inconsistent."],
                    "required_research": ["Resolve 41 versus 42."],
                    "rerun_plan": False,
                },
                {
                    "status": "pass",
                    "answer": "42",
                    "supporting_evidence": ["Two sources establish 42."],
                    "issues": [],
                    "required_research": [],
                    "rerun_plan": False,
                },
            ]
        )
        system.finalizer = _FakeStructuredAgent([{"answer": "42"}])

        answer = system.solve("task", "What is the count?", "No attachment.")

        self.assertEqual(answer, "42")
        self.assertEqual(system.researcher.calls, 2)

    def test_best_effort_result_is_explicitly_unvalidated(self) -> None:
        system = LocalAgentSystem.__new__(LocalAgentSystem)
        system.settings = AgentSettings(
            ollama_base_url="http://localhost:11434",
            text_model=DEFAULT_OLLAMA_MODEL,
            multimodal_model=DEFAULT_OLLAMA_MODEL,
            context_size=8192,
            max_research_steps=2,
            max_validation_retries=0,
        )
        plan = {
            "answer_format": "integer",
            "facts_to_verify": ["the exact count"],
            "research_queries": ["authoritative count"],
            "calculations": [],
            "attachment_use": "none",
        }
        system.planner = _FakeStructuredAgent([plan])
        system.researcher = _FakeResearcher(["A weak source suggests 41."])
        system.validator = _FakeStructuredAgent(
            [
                {
                    "status": "retry",
                    "answer": "41",
                    "supporting_evidence": [],
                    "issues": ["The count is not independently supported."],
                    "required_research": ["Find an authoritative source."],
                    "rerun_plan": False,
                }
            ]
        )
        system.finalizer = _FakeStructuredAgent([{"answer": "41"}])

        result = system.solve_result(
            "task",
            "What is the count?",
            "No attachment.",
            allow_best_effort=True,
        )

        self.assertEqual(result.answer, "41")
        self.assertFalse(result.validated)
        self.assertEqual(result.issues, ["The count is not independently supported."])


class CleanSubmissionValueTests(unittest.TestCase):
    def test_extracts_validator_marker(self) -> None:
        self.assertEqual(
            clean_submission_value("SUBMISSION_VALUE: 42"),
            "42",
        )

    def test_removes_thinking_and_quotes(self) -> None:
        raw = '<think>private reasoning</think>\nSUBMISSION_VALUE: "Ada Lovelace"'
        self.assertEqual(clean_submission_value(raw), "Ada Lovelace")

    def test_rejects_multiline_answer(self) -> None:
        with self.assertRaises(ValueError):
            clean_submission_value("first line\nsecond line")

    def test_rejects_forbidden_phrase(self) -> None:
        with self.assertRaises(ValueError):
            clean_submission_value("The final answer is 42")

    def test_removes_extraneous_single_word_punctuation(self) -> None:
        self.assertEqual(clean_submission_value("Extremely."), "Extremely")

    def test_detects_refusal_placeholders(self) -> None:
        self.assertTrue(is_placeholder_answer("N/A"))
        self.assertTrue(is_placeholder_answer("Cannot determine"))
        self.assertTrue(is_placeholder_answer("}<tool_call|>"))
        self.assertFalse(is_placeholder_answer("St. Petersburg"))


class QuestionTransformTests(unittest.TestCase):
    def test_decodes_fully_reversed_question(self) -> None:
        encoded = '.rewsna eht sa "tfel" drow eht fo etisoppo eht etirw'
        hint = question_transform_hints(encoded)

        self.assertIn('write the opposite of the word "left" as the answer.', hint)

    def test_leaves_normal_question_unchanged(self) -> None:
        self.assertEqual(question_transform_hints("What is 2 + 2?"), "")


if __name__ == "__main__":
    unittest.main()