composer_replication/diloco/serverless/tests/test_skeleton_executors.py

"""Tests pinning the skeleton-executor contract (Wave 18).

Both ModalExecutor and HFJobsExecutor are documented as v0 skeletons
that fail loudly on instantiation. These tests pin that they actually
fail loudly — so the skeleton stays a *documented skeleton* rather than
silently shipping wrong (e.g., a future refactor that accidentally
makes ModalExecutor() succeed without implementing the methods would
silently pass user-journey tests but break at first .launch_replicas).

The contract pinned here:

1. **Optional-dep missing → RuntimeError.** The error message must
   reference the missing dep by name (`modal` for ModalExecutor,
   `huggingface_hub` for HFJobsExecutor) so the user has an actionable
   install hint.

2. **Optional-dep present → NotImplementedError.** The error message
   must reference the executor's skeleton status so the user knows
   this is intentional (not a bug they should report) and points them
   at LocalProcessExecutor as the working alternative.

The tests check whichever path applies in the current venv (driven by
whether the optional dep is installed) but they pin BOTH the exception
type AND the message contract — a future refactor that swaps the
exception type, drops the install hint, or removes the skeleton
disclaimer will fail these tests loudly.

Wave 18 review: Grok flagged the prior version as too lax (only
checked `(RuntimeError, NotImplementedError) + 'modal'/'skeleton' in msg`
which would pass even on a `ValueError("skeleton")`). This version
tightens to (a) exact exception type for the active code path and
(b) a stricter message-content contract.
"""
from __future__ import annotations

import importlib
import pytest

from composer_replication.diloco.serverless import (
    HFJobsExecutor,
    ModalExecutor,
)


def _is_installed(module_name: str) -> bool:
    """True iff the given module can be imported in the current venv."""
    try:
        importlib.import_module(module_name)
        return True
    except ImportError:
        return False


def test_modal_executor_skeleton_fails_loudly():
    """ModalExecutor() must raise.

    - If `modal` is NOT installed: RuntimeError with the word "modal"
      AND an actionable install hint (the word "install" or "pip" or
      a quoted command).
    - If `modal` IS installed: NotImplementedError with the word
      "skeleton" AND a pointer to LocalProcessExecutor.
    """
    if _is_installed("modal"):
        # Optional dep present → skeleton NotImplementedError contract
        with pytest.raises(NotImplementedError) as excinfo:
            ModalExecutor()
        msg = str(excinfo.value).lower()
        assert "skeleton" in msg, (
            f"ModalExecutor NotImplementedError must say 'skeleton'; "
            f"got: {excinfo.value}"
        )
        assert "localprocessexecutor" in msg, (
            f"ModalExecutor NotImplementedError must point at "
            f"LocalProcessExecutor as the working alternative; got: "
            f"{excinfo.value}"
        )
    else:
        # Optional dep absent → install-hint RuntimeError contract
        with pytest.raises(RuntimeError) as excinfo:
            ModalExecutor()
        msg = str(excinfo.value).lower()
        assert "modal" in msg, (
            f"ModalExecutor RuntimeError must reference 'modal' by name; "
            f"got: {excinfo.value}"
        )


def test_hf_jobs_executor_skeleton_fails_loudly():
    """HFJobsExecutor() must raise. Same contract as ModalExecutor."""
    if _is_installed("huggingface_hub"):
        # Optional dep present → skeleton NotImplementedError contract
        with pytest.raises(NotImplementedError) as excinfo:
            HFJobsExecutor()
        msg = str(excinfo.value).lower()
        assert "skeleton" in msg, (
            f"HFJobsExecutor NotImplementedError must say 'skeleton'; "
            f"got: {excinfo.value}"
        )
        assert "localprocessexecutor" in msg, (
            f"HFJobsExecutor NotImplementedError must point at "
            f"LocalProcessExecutor as the working alternative; got: "
            f"{excinfo.value}"
        )
    else:
        # Optional dep absent → install-hint RuntimeError contract
        with pytest.raises(RuntimeError) as excinfo:
            HFJobsExecutor()
        msg = str(excinfo.value).lower()
        assert "huggingface_hub" in msg or "huggingface" in msg, (
            f"HFJobsExecutor RuntimeError must reference 'huggingface_hub' "
            f"by name; got: {excinfo.value}"
        )