Upload 34 files

DEPLOY.md

@@ -75,14 +75,19 @@ real LLM responses:
    picks up new secrets on the next restart).
 
 **Note on the API endpoint:** `src/llm_client.py` explicitly passes
-`provider="hf-inference"` to `InferenceClient`, and `requirements.txt`
-pins a recent `huggingface_hub` version. Both matter: older client
-versions / omitting the provider can silently route calls through the
-now-deprecated `api-inference.huggingface.co` domain, which fails with a
-DNS resolution error rather than a clear auth error. If you ever see a
-`NameResolutionError` mentioning `api-inference.huggingface.co` in the
-"Test LLM connection" diagnostics, bump `huggingface_hub` in
-`requirements.txt` to the latest release.
+`provider="auto"` to `InferenceClient`, and `requirements.txt` pins a
+recent `huggingface_hub` version. Both matter:
+- Older client versions / omitting the provider can silently route calls
+  through the now-deprecated `api-inference.huggingface.co` domain, which
+  fails with a DNS resolution error rather than a clear auth error.
+- Hardcoding a specific provider (e.g. `"hf-inference"`) can fail with
+  *"Model not supported by provider ..."* for models that are actually
+  hosted via a different backend (Together, Fireworks, Novita, SambaNova,
+  etc.) under HF's Inference Providers system. `provider="auto"` lets HF's
+  router pick whichever backend actually serves each candidate model.
+
+If you ever see errors in the "Test LLM connection" diagnostics, they'll
+show you exactly which of the above it is per candidate model.
 
 ## Re-training / updating the models
 

src/llm_client.py

@@ -34,8 +34,10 @@ from src.retriever import KBRetriever, RetrievedDoc
 # LLM_MODEL_ID is set as an env var, it is tried first, ahead of this list.
 MODEL_CANDIDATES = [
     "Qwen/Qwen2.5-7B-Instruct",
+    "meta-llama/Llama-3.1-8B-Instruct",
     "meta-llama/Llama-3.2-3B-Instruct",
     "mistralai/Mistral-7B-Instruct-v0.3",
+    "microsoft/Phi-3.5-mini-instruct",
     "HuggingFaceH4/zephyr-7b-beta",
 ]
 
@@ -125,12 +127,12 @@ def answer_query(
     errors = []
     for candidate in MODEL_CANDIDATES:
         try:
-            # `provider="hf-inference"` forces routing through HF's current
-            # Inference Providers router (router.huggingface.co). Without it,
-            # older huggingface_hub versions / ambiguous setups can fall back
-            # to the deprecated `api-inference.huggingface.co` endpoint,
-            # which has been sunset and fails with a DNS resolution error.
-            client = InferenceClient(model=candidate, token=hf_token, provider="hf-inference")
+            # provider="auto" (the huggingface_hub default) lets HF's Inference
+            # Providers router pick whichever backend (hf-inference, Together,
+            # Fireworks, Novita, SambaNova, etc.) actually serves this specific
+            # model -- hardcoding a single provider caused "model not supported
+            # by provider X" errors for models hosted elsewhere.
+            client = InferenceClient(model=candidate, token=hf_token, provider="auto")
             completion = client.chat_completion(messages=messages, max_tokens=max_tokens, temperature=0.3)
             text = completion.choices[0].message.content
             if text and text.strip():