app inference gradio config models

apps/gradio-space/src/gradio_space/app.py

@@ -101,8 +101,6 @@ Part of the [Build Small Hackathon](https://huggingface.co/build-small-hackathon
 """
         )
 
-        model_key = gr.State(_app_config.active_model)
-
         if _app_config.allow_model_switch and len(_app_config.models) > 1:
             model_dropdown = gr.Dropdown(
                 choices=_app_config.model_choices(),
@@ -116,10 +114,6 @@ Part of the [Build Small Hackathon](https://huggingface.co/build-small-hackathon
                 fn=model_status,
                 inputs=model_dropdown,
                 outputs=status,
-            ).then(
-                fn=lambda key: key,
-                inputs=model_dropdown,
-                outputs=model_key,
             )
 
             gr.ChatInterface(

libs/inference/src/inference/__init__.py

@@ -1,3 +1,12 @@
-from inference.factory import get_backend
+from inference.config import AppConfig, ModelConfig, get_app_config, get_model_config, load_app_config
+from inference.factory import get_backend, reset_backend
 
-__all__ = ["get_backend"]
+__all__ = [
+    "AppConfig",
+    "ModelConfig",
+    "get_app_config",
+    "get_backend",
+    "get_model_config",
+    "load_app_config",
+    "reset_backend",
+]

libs/inference/src/inference/config.py

@@ -0,0 +1,281 @@
+"""Central model preset and app configuration."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, replace
+from pathlib import Path
+from typing import Any, Literal
+
+BackendName = Literal["llama_cpp", "transformers"]
+
+DEFAULT_PRESET_KEY = "qwen3b-gguf"
+
+
+@dataclass(frozen=True)
+class ModelConfig:
+    """Single model preset used by inference backends and the Gradio UI."""
+
+    key: str
+    label: str
+    backend: BackendName
+    model_repo: str | None = None
+    model_file: str | None = None
+    model_path: str | None = None
+    model_id: str | None = None
+    trust_remote_code: bool = False
+    n_ctx: int = 4096
+    n_gpu_layers: int = 0
+    max_tokens: int = 512
+    temperature: float = 0.7
+
+    def cache_key(self) -> tuple[Any, ...]:
+        return (
+            self.backend,
+            self.model_repo,
+            self.model_file,
+            self.model_path,
+            self.model_id,
+            self.trust_remote_code,
+            self.n_ctx,
+            self.n_gpu_layers,
+        )
+
+    def summary(self) -> str:
+        if self.backend == "llama_cpp":
+            source = self.model_path or f"{self.model_repo}/{self.model_file}"
+            return f"{self.label} · llama.cpp · {source}"
+        return f"{self.label} · transformers · {self.model_id}"
+
+    def resolve_paths(self, base_dir: Path) -> ModelConfig:
+        updates: dict[str, Any] = {}
+
+        if self.model_path:
+            path = Path(self.model_path)
+            if not path.is_absolute():
+                updates["model_path"] = str((base_dir / path).resolve())
+
+        if self.model_id and self.model_id.startswith(("./", "../")):
+            updates["model_id"] = str((base_dir / self.model_id).resolve())
+
+        return replace(self, **updates) if updates else self
+
+
+@dataclass(frozen=True)
+class AppConfig:
+    """Runtime app configuration for dev and Hugging Face Space."""
+
+    active_model: str
+    models: dict[str, ModelConfig]
+    allow_model_switch: bool = False
+    model_cache_dir: str | None = None
+    presets_path: Path | None = None
+
+    def get_model(self, key: str | None = None) -> ModelConfig:
+        model_key = key or self.active_model
+        if model_key not in self.models:
+            known = ", ".join(sorted(self.models))
+            raise KeyError(f"Unknown model preset {model_key!r}. Known presets: {known}")
+        return self.models[model_key]
+
+    @property
+    def active(self) -> ModelConfig:
+        return self.get_model(self.active_model)
+
+    def model_choices(self) -> list[tuple[str, str]]:
+        return [(model.label, model.key) for model in self.models.values()]
+
+
+def _builtin_presets() -> dict[str, ModelConfig]:
+    return {
+        DEFAULT_PRESET_KEY: ModelConfig(
+            key=DEFAULT_PRESET_KEY,
+            label="Qwen 2.5 3B Instruct (GGUF)",
+            backend="llama_cpp",
+            model_repo="Qwen/Qwen2.5-3B-Instruct-GGUF",
+            model_file="qwen2.5-3b-instruct-q4_k_m.gguf",
+        ),
+        "minicpm5-1b": ModelConfig(
+            key="minicpm5-1b",
+            label="MiniCPM5 1B (Transformers)",
+            backend="transformers",
+            model_id="openbmb/MiniCPM5-1B",
+            trust_remote_code=True,
+        ),
+        "gemma-merged-local": ModelConfig(
+            key="gemma-merged-local",
+            label="Fine-tuned merged model (local)",
+            backend="transformers",
+            model_id="./gemma_merged_model",
+        ),
+    }
+
+
+def _find_presets_path() -> Path | None:
+    env_path = os.environ.get("MODEL_PRESETS_PATH")
+    if env_path:
+        path = Path(env_path)
+        if path.is_file():
+            return path.resolve()
+
+    for base in (Path.cwd(), *Path.cwd().parents):
+        candidate = base / "models.yaml"
+        if candidate.is_file():
+            return candidate.resolve()
+    return None
+
+
+def _repo_root_for(presets_path: Path | None) -> Path:
+    if presets_path is not None:
+        return presets_path.parent
+    app_root = os.environ.get("APP_ROOT")
+    if app_root:
+        return Path(app_root).resolve()
+    return Path.cwd().resolve()
+
+
+def _parse_model_entry(key: str, raw: dict[str, Any]) -> ModelConfig:
+    backend = str(raw.get("backend", "llama_cpp")).lower()
+    if backend not in ("llama_cpp", "transformers"):
+        raise ValueError(f"Preset {key!r}: backend must be llama_cpp or transformers")
+
+    return ModelConfig(
+        key=key,
+        label=str(raw.get("label", key)),
+        backend=backend,  # type: ignore[arg-type]
+        model_repo=raw.get("model_repo"),
+        model_file=raw.get("model_file"),
+        model_path=raw.get("model_path"),
+        model_id=raw.get("model_id"),
+        trust_remote_code=bool(raw.get("trust_remote_code", False)),
+        n_ctx=int(raw.get("n_ctx", 4096)),
+        n_gpu_layers=int(raw.get("n_gpu_layers", 0)),
+        max_tokens=int(raw.get("max_tokens", 512)),
+        temperature=float(raw.get("temperature", 0.7)),
+    )
+
+
+def _load_presets_from_yaml(path: Path) -> tuple[dict[str, Any], dict[str, ModelConfig]]:
+    try:
+        import yaml
+    except ImportError as exc:
+        raise ImportError(
+            "Loading models.yaml requires PyYAML. Install with: uv add --package inference pyyaml"
+        ) from exc
+
+    data = yaml.safe_load(path.read_text()) or {}
+    defaults = data.get("defaults", {})
+    raw_models = data.get("models", {})
+    if not isinstance(raw_models, dict) or not raw_models:
+        raise ValueError(f"{path}: expected non-empty top-level 'models' mapping")
+
+    models = {key: _parse_model_entry(key, value) for key, value in raw_models.items()}
+    return defaults, models
+
+
+def _apply_legacy_env_overrides(model: ModelConfig) -> ModelConfig:
+    """Keep single-model .env workflow working alongside preset keys."""
+
+    updates: dict[str, Any] = {}
+
+    backend = os.environ.get("INFERENCE_BACKEND")
+    if backend:
+        updates["backend"] = backend.lower()
+
+    for field, env_name in (
+        ("model_repo", "MODEL_REPO"),
+        ("model_file", "MODEL_FILE"),
+        ("model_path", "MODEL_PATH"),
+        ("model_id", "MODEL_ID"),
+    ):
+        value = os.environ.get(env_name)
+        if value:
+            updates[field] = value
+
+    if os.environ.get("TRUST_REMOTE_CODE", "").lower() in {"1", "true", "yes"}:
+        updates["trust_remote_code"] = True
+
+    for field, env_name in (
+        ("n_ctx", "N_CTX"),
+        ("n_gpu_layers", "N_GPU_LAYERS"),
+        ("max_tokens", "MAX_TOKENS"),
+    ):
+        value = os.environ.get(env_name)
+        if value is not None and value != "":
+            updates[field] = int(value)
+
+    temperature = os.environ.get("TEMPERATURE")
+    if temperature is not None and temperature != "":
+        updates["temperature"] = float(temperature)
+
+    return replace(model, **updates) if updates else model
+
+
+def load_app_config() -> AppConfig:
+    presets_path = _find_presets_path()
+    repo_root = _repo_root_for(presets_path)
+
+    if presets_path is None:
+        defaults: dict[str, Any] = {}
+        models = _builtin_presets()
+    else:
+        defaults, models = _load_presets_from_yaml(presets_path)
+
+    active_model = os.environ.get("ACTIVE_MODEL") or defaults.get(
+        "active_model", DEFAULT_PRESET_KEY
+    )
+    if active_model not in models:
+        active_model = next(iter(models))
+
+    allow_model_switch = os.environ.get("ALLOW_MODEL_SWITCH")
+    if allow_model_switch is None:
+        allow_switch = bool(defaults.get("allow_model_switch", False))
+    else:
+        allow_switch = allow_model_switch.lower() in {"1", "true", "yes"}
+
+    cache_dir = os.environ.get("MODEL_CACHE_DIR") or defaults.get("model_cache_dir")
+
+    resolved_models = {
+        key: model.resolve_paths(repo_root) for key, model in models.items()
+    }
+
+    has_legacy_override = any(
+        os.environ.get(name)
+        for name in (
+            "INFERENCE_BACKEND",
+            "MODEL_REPO",
+            "MODEL_FILE",
+            "MODEL_PATH",
+            "MODEL_ID",
+            "N_CTX",
+            "N_GPU_LAYERS",
+        )
+    )
+    if has_legacy_override:
+        resolved_models[active_model] = _apply_legacy_env_overrides(
+            resolved_models[active_model]
+        )
+
+    return AppConfig(
+        active_model=active_model,
+        models=resolved_models,
+        allow_model_switch=allow_switch,
+        model_cache_dir=cache_dir,
+        presets_path=presets_path,
+    )
+
+
+_app_config: AppConfig | None = None
+
+
+def get_app_config(reload: bool = False) -> AppConfig:
+    global _app_config
+    if _app_config is None or reload:
+        _app_config = load_app_config()
+    return _app_config
+
+
+def get_model_config(key: str | None = None) -> ModelConfig:
+    config = get_app_config()
+    model_key = key or config.active_model
+    return config.get_model(model_key)

libs/inference/src/inference/factory.py

@@ -1,23 +1,40 @@
-import os
-from functools import lru_cache
-
 from inference.base import InferenceBackend
+from inference.config import ModelConfig, get_model_config
 from inference.llama_cpp import LlamaCppBackend
 
+_backend: InferenceBackend | None = None
+_backend_key: tuple | None = None
 
-@lru_cache(maxsize=1)
-def get_backend() -> InferenceBackend:
-    backend_name = os.environ.get("INFERENCE_BACKEND", "llama_cpp").lower()
 
-    if backend_name == "llama_cpp":
-        return LlamaCppBackend()
+def _create_backend(config: ModelConfig) -> InferenceBackend:
+    if config.backend == "llama_cpp":
+        return LlamaCppBackend(config)
 
-    if backend_name == "transformers":
+    if config.backend == "transformers":
         from inference.transformers import TransformersBackend
 
-        return TransformersBackend()
+        return TransformersBackend(config)
 
     raise ValueError(
-        f"Unknown INFERENCE_BACKEND={backend_name!r}. "
+        f"Unknown backend {config.backend!r} for preset {config.key!r}. "
         "Expected 'llama_cpp' or 'transformers'."
     )
+
+
+def get_backend(model_key: str | None = None) -> InferenceBackend:
+    global _backend, _backend_key
+
+    config = get_model_config(model_key)
+    cache_key = config.cache_key()
+
+    if _backend is None or _backend_key != cache_key:
+        _backend = _create_backend(config)
+        _backend_key = cache_key
+
+    return _backend
+
+
+def reset_backend() -> None:
+    global _backend, _backend_key
+    _backend = None
+    _backend_key = None

libs/inference/src/inference/llama_cpp.py

@@ -4,31 +4,32 @@ from pathlib import Path
 from huggingface_hub import hf_hub_download
 from llama_cpp import Llama
 
-
-DEFAULT_MODEL_REPO = "Qwen/Qwen2.5-3B-Instruct-GGUF"
-DEFAULT_MODEL_FILE = "qwen2.5-3b-instruct-q4_k_m.gguf"
+from inference.config import ModelConfig
 
 
 class LlamaCppBackend:
-    def __init__(self) -> None:
+    def __init__(self, config: ModelConfig) -> None:
+        self._config = config
         self._model: Llama | None = None
         self._model_path: str | None = None
 
     def _resolve_model_path(self) -> str:
-        model_path = os.environ.get("MODEL_PATH")
-        if model_path:
-            path = Path(model_path)
+        if self._config.model_path:
+            path = Path(self._config.model_path)
             if not path.exists():
-                raise FileNotFoundError(f"MODEL_PATH does not exist: {model_path}")
+                raise FileNotFoundError(f"MODEL_PATH does not exist: {self._config.model_path}")
             return str(path)
 
-        model_repo = os.environ.get("MODEL_REPO", DEFAULT_MODEL_REPO)
-        model_file = os.environ.get("MODEL_FILE", DEFAULT_MODEL_FILE)
+        if not self._config.model_repo or not self._config.model_file:
+            raise ValueError(
+                f"Preset {self._config.key!r} requires model_repo and model_file for llama_cpp"
+            )
+
         cache_dir = os.environ.get("MODEL_CACHE_DIR")
 
         return hf_hub_download(
-            repo_id=model_repo,
-            filename=model_file,
+            repo_id=self._config.model_repo,
+            filename=self._config.model_file,
             cache_dir=cache_dir,
         )
 
@@ -37,13 +38,11 @@ class LlamaCppBackend:
             return
 
         self._model_path = self._resolve_model_path()
-        n_ctx = int(os.environ.get("N_CTX", "4096"))
-        n_gpu_layers = int(os.environ.get("N_GPU_LAYERS", "0"))
 
         self._model = Llama(
             model_path=self._model_path,
-            n_ctx=n_ctx,
-            n_gpu_layers=n_gpu_layers,
+            n_ctx=self._config.n_ctx,
+            n_gpu_layers=self._config.n_gpu_layers,
             verbose=False,
         )
 
@@ -51,16 +50,16 @@ class LlamaCppBackend:
         self,
         prompt: str,
         *,
-        max_tokens: int = 512,
-        temperature: float = 0.7,
+        max_tokens: int | None = None,
+        temperature: float | None = None,
     ) -> str:
         self.load()
         assert self._model is not None
 
         result = self._model(
             prompt,
-            max_tokens=max_tokens,
-            temperature=temperature,
+            max_tokens=max_tokens or self._config.max_tokens,
+            temperature=temperature if temperature is not None else self._config.temperature,
             echo=False,
         )
         return result["choices"][0]["text"].strip()
@@ -69,15 +68,15 @@ class LlamaCppBackend:
         self,
         messages: list[dict[str, str]],
         *,
-        max_tokens: int = 512,
-        temperature: float = 0.7,
+        max_tokens: int | None = None,
+        temperature: float | None = None,
     ) -> str:
         self.load()
         assert self._model is not None
 
         result = self._model.create_chat_completion(
             messages=messages,
-            max_tokens=max_tokens,
-            temperature=temperature,
+            max_tokens=max_tokens or self._config.max_tokens,
+            temperature=temperature if temperature is not None else self._config.temperature,
         )
         return result["choices"][0]["message"]["content"].strip()

libs/inference/src/inference/transformers.py

@@ -1,10 +1,9 @@
-import os
-
-from inference.base import InferenceBackend
+from inference.config import ModelConfig
 
 
 class TransformersBackend:
-    def __init__(self) -> None:
+    def __init__(self, config: ModelConfig) -> None:
+        self._config = config
         self._model = None
         self._tokenizer = None
 
@@ -12,6 +11,11 @@ class TransformersBackend:
         if self._model is not None:
             return
 
+        if not self._config.model_id:
+            raise ValueError(
+                f"Preset {self._config.key!r} requires model_id for transformers backend"
+            )
+
         try:
             import torch
             from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -21,14 +25,17 @@ class TransformersBackend:
                 "Install with: uv sync --package inference --extra transformers"
             ) from exc
 
-        model_id = os.environ.get("MODEL_ID", "Qwen/Qwen2.5-3B-Instruct")
         device = "cuda" if torch.cuda.is_available() else "cpu"
 
-        self._tokenizer = AutoTokenizer.from_pretrained(model_id)
+        self._tokenizer = AutoTokenizer.from_pretrained(
+            self._config.model_id,
+            trust_remote_code=self._config.trust_remote_code,
+        )
         self._model = AutoModelForCausalLM.from_pretrained(
-            model_id,
+            self._config.model_id,
             torch_dtype=torch.float16 if device == "cuda" else torch.float32,
             device_map="auto" if device == "cuda" else None,
+            trust_remote_code=self._config.trust_remote_code,
         )
         if device == "cpu":
             self._model.to(device)
@@ -37,8 +44,8 @@ class TransformersBackend:
         self,
         prompt: str,
         *,
-        max_tokens: int = 512,
-        temperature: float = 0.7,
+        max_tokens: int | None = None,
+        temperature: float | None = None,
     ) -> str:
         self.load()
         assert self._model is not None
@@ -46,12 +53,15 @@ class TransformersBackend:
 
         import torch
 
+        max_new_tokens = max_tokens or self._config.max_tokens
+        temp = self._config.temperature if temperature is None else temperature
+
         inputs = self._tokenizer(prompt, return_tensors="pt").to(self._model.device)
         output = self._model.generate(
             **inputs,
-            max_new_tokens=max_tokens,
-            temperature=temperature,
-            do_sample=temperature > 0,
+            max_new_tokens=max_new_tokens,
+            temperature=temp,
+            do_sample=temp > 0,
         )
         generated = output[0][inputs["input_ids"].shape[-1] :]
         return self._tokenizer.decode(generated, skip_special_tokens=True).strip()
@@ -60,8 +70,8 @@ class TransformersBackend:
         self,
         messages: list[dict[str, str]],
         *,
-        max_tokens: int = 512,
-        temperature: float = 0.7,
+        max_tokens: int | None = None,
+        temperature: float | None = None,
     ) -> str:
         self.load()
         assert self._model is not None
@@ -83,7 +93,3 @@ class TransformersBackend:
             prompt = "\n".join(parts)
 
         return self.generate(prompt, max_tokens=max_tokens, temperature=temperature)
-
-
-# Satisfy static type checkers that expect InferenceBackend.
-_: InferenceBackend = TransformersBackend()

libs/inference/tests/test_config.py

@@ -0,0 +1,76 @@
+from pathlib import Path
+
+import pytest
+
+from inference.config import load_app_config
+
+
+def test_load_app_config_from_models_yaml(tmp_path, monkeypatch):
+    presets = tmp_path / "models.yaml"
+    presets.write_text(
+        """
+defaults:
+  active_model: demo
+  allow_model_switch: true
+models:
+  demo:
+    label: Demo preset
+    backend: llama_cpp
+    model_repo: org/model-GGUF
+    model_file: demo.gguf
+"""
+    )
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.delenv("ACTIVE_MODEL", raising=False)
+
+    config = load_app_config()
+
+    assert config.active_model == "demo"
+    assert config.allow_model_switch is True
+    assert config.get_model("demo").model_repo == "org/model-GGUF"
+
+
+def test_legacy_env_overrides_active_preset(tmp_path, monkeypatch):
+    presets = tmp_path / "models.yaml"
+    presets.write_text(
+        """
+defaults:
+  active_model: demo
+models:
+  demo:
+    label: Demo
+    backend: llama_cpp
+    model_repo: org/original
+    model_file: original.gguf
+"""
+    )
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setenv("MODEL_REPO", "org/override")
+    monkeypatch.setenv("MODEL_FILE", "override.gguf")
+
+    model = load_app_config().get_model("demo")
+
+    assert model.model_repo == "org/override"
+    assert model.model_file == "override.gguf"
+
+
+def test_resolve_relative_model_path(tmp_path, monkeypatch):
+    local_dir = tmp_path / "gemma_merged_model"
+    local_dir.mkdir()
+    presets = tmp_path / "models.yaml"
+    presets.write_text(
+        f"""
+defaults:
+  active_model: local
+models:
+  local:
+    label: Local merged
+    backend: transformers
+    model_id: ./{local_dir.name}
+"""
+    )
+    monkeypatch.chdir(tmp_path)
+
+    model = load_app_config().get_model("local")
+
+    assert model.model_id == str(local_dir.resolve())

scripts/download_model.py

@@ -1,26 +1,22 @@
 #!/usr/bin/env python3
-"""Download the configured GGUF model from Hugging Face Hub for offline dev."""
+"""Download a configured GGUF preset from Hugging Face Hub for offline dev."""
 
 from __future__ import annotations
 
 import argparse
-import os
 from pathlib import Path
 
 from huggingface_hub import hf_hub_download
 
+from inference.config import get_app_config, get_model_config
+
 
 def main() -> None:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument(
-        "--repo",
-        default=os.environ.get("MODEL_REPO", "Qwen/Qwen2.5-3B-Instruct-GGUF"),
-        help="Hugging Face repo containing the GGUF file",
-    )
-    parser.add_argument(
-        "--file",
-        default=os.environ.get("MODEL_FILE", "qwen2.5-3b-instruct-q4_k_m.gguf"),
-        help="GGUF filename inside the repo",
+        "--preset",
+        default=None,
+        help="Preset key from models.yaml (default: ACTIVE_MODEL or app default)",
     )
     parser.add_argument(
         "--output-dir",
@@ -30,16 +26,39 @@ def main() -> None:
     )
     args = parser.parse_args()
 
+    app_config = get_app_config()
+    preset_key = args.preset or app_config.active_model
+    model = get_model_config(preset_key)
+
+    if model.backend != "llama_cpp":
+        raise SystemExit(
+            f"Preset {preset_key!r} uses backend {model.backend!r}. "
+            "Only llama_cpp presets with model_repo/model_file can be downloaded."
+        )
+
+    if model.model_path:
+        path = Path(model.model_path)
+        if not path.exists():
+            raise SystemExit(f"Local MODEL_PATH does not exist: {model.model_path}")
+        print(f"Preset {preset_key!r} already points to local file: {path}")
+        print(f"Set MODEL_PATH={path} or update models.yaml model_path to use it directly.")
+        return
+
+    if not model.model_repo or not model.model_file:
+        raise SystemExit(f"Preset {preset_key!r} is missing model_repo/model_file.")
+
     args.output_dir.mkdir(parents=True, exist_ok=True)
 
     path = hf_hub_download(
-        repo_id=args.repo,
-        filename=args.file,
+        repo_id=model.model_repo,
+        filename=model.model_file,
         local_dir=args.output_dir,
         local_dir_use_symlinks=False,
     )
+    print(f"Preset: {preset_key} ({model.label})")
     print(f"Model ready at: {path}")
-    print(f"Set MODEL_PATH={path} to use this file directly.")
+    print("Add to models.yaml under that preset:")
+    print(f"  model_path: {Path(path).resolve()}")
 
 
 if __name__ == "__main__":