Initial release: 7-domain LSTM prediction system with chat interface

README.md

@@ -0,0 +1,89 @@
+---
+license: mit
+library_name: pytorch
+tags:
+- time-series
+- forecasting
+- lstm
+- finance
+- weather
+- multi-domain
+language:
+- en
+metrics:
+- mae
+- mape
+pipeline_tag: time-series-forecasting
+---
+
+# Future Prediction Models (Multi-Domain LSTM)
+
+An end-to-end multi-domain forecasting system. For each of **7 topics** it automatically fetches a real dataset, trains a 2-layer LSTM, evaluates on a held-out test set (no leakage), and generates ChatGPT-style future predictions.
+
+## Topics & validated accuracy
+
+| Topic | Asset | Source | H1 MAPE | vs naive |
+|---|---|---|---|---|
+| AI | NVIDIA daily close | Yahoo Finance | 1.86% | ~naive |
+| Economy | S&P 500 daily close | Yahoo Finance | 0.64% | beats naive 1.8% |
+| Energy | WTI crude oil close | Yahoo Finance | 2.83% | ~naive |
+| Finance | Bitcoin BTC-USD | Yahoo Finance | 1.52% | ~naive |
+| Programming | `react` npm downloads | npm registry | 7.12% | **beats naive 77.5%** |
+| Sports | ATP world #1 Elo | Hugging Face tennis | 0.06% | ~naive |
+| Weather | Daily mean temperature | Open-Meteo | 0.18% | beats naive 1.9% |
+
+> Markets behave near a random walk, so 100% accuracy is impossible — these are honest, validated numbers.
+
+## Quick start
+
+```bash
+pip install -r requirements.txt
+python data.py --topic finance --refresh   # fetch real data
+python train.py --topic finance            # train the LSTM
+python ask.py "what will bitcoin do next week?"
+python ask.py "compare all topics"
+```
+
+## Architecture
+
+```
+user question -> intent detection (response.py)
+              -> prediction_engine.py   (structured metrics, prints nothing)
+              -> response_formatter.py  (natural summary + Markdown)
+              -> ask.py                 (chat CLI, stdout)
+```
+
+The engine returns clearly separated metrics — never conflating them:
+- `net_change_pct` (first → last forecast along the path)
+- `forecast_range_pct` (max − min of the forecast path)
+- `current_to_forecast_pct` (final forecast vs latest observed value)
+- `direction` (rising / falling / sideways, configurable threshold)
+
+Only the final assistant response reaches the user; internal logs go to stderr and only when `FORECAST_DEBUG=1`.
+
+## Files
+
+- `model_<topic>.pt` — trained PyTorch LSTM checkpoint per topic
+- `config_<topic>.json` — hyperparameters + validation metrics per topic
+- `data.py`, `train.py`, `predict.py`, `model.py` — data + training + inference core
+- `ask.py`, `response.py`, `response_formatter.py`, `prediction_engine.py`, `debug_logger.py` — the chat/response layer
+- `weather_cities.py` — city registry for location-specific weather models
+
+## Weather for any city
+
+```bash
+export WEATHER_NAME='Chennai, India' WEATHER_LAT=13.0827 WEATHER_LON=80.2707
+python data.py --topic weather --refresh
+python train.py --topic weather
+python ask.py "predict weather in chennai tomorrow"
+```
+
+## Add a new domain
+
+1. Add an entry to `TOPICS` in `data.py` (label, unit, asset)
+2. Add a fetcher returning `{source, series}` to `FETCHERS`
+3. Run `python train.py --topic <name>` — everything else is automatic
+
+## License
+
+MIT

ask.py

@@ -0,0 +1,69 @@
+"""Chat interface. Only the final assistant response reaches stdout.
+
+Usage:
+  python ask.py                                  # interactive chat
+  python ask.py "what will bitcoin do next week?"
+  python ask.py --debug "compare all topics"     # internal logs on stderr
+"""
+
+import argparse
+import sys
+import time
+
+import debug_logger
+import response
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("question", nargs="*", help="one-shot question (omit for interactive chat)")
+    parser.add_argument("--stream", action="store_true", help="stream output line by line")
+    parser.add_argument("--debug", action="store_true", help="show internal logs on stderr")
+    parser.add_argument("--detailed", action="store_true", help="include day-by-day table")
+    parser.add_argument("--device", default="cpu")
+    args = parser.parse_args()
+
+    if args.debug:
+        import os
+        os.environ["FORECAST_DEBUG"] = "1"
+
+    config = response.GenerationConfig(debug=args.debug, detailed=args.detailed)
+    ctx = response.ConversationContext(max_turns=8)
+
+    def ask(question: str) -> str:
+        answer = debug_logger.strip_internal(
+            response.generate_response(question, device=args.device, config=config))
+        if args.stream:
+            for chunk in response.stream_response(question, pregenerated=answer):
+                sys.stdout.write(chunk)
+                sys.stdout.flush()
+                time.sleep(config.chunk_delay)
+            if answer and not answer.endswith("\n"):
+                print()
+        else:
+            print(answer)
+        ctx.add("user", question)
+        ctx.add("assistant", answer)
+        return answer
+
+    if args.question:
+        ask(" ".join(args.question))
+        return
+
+    print("Forecast assistant - ask about AI, Programming, Finance, Sports,")
+    print("Weather, Economy, or Energy. Example: 'what will bitcoin do next week?'")
+    while True:
+        try:
+            q = input("you> ").strip()
+        except (EOFError, KeyboardInterrupt):
+            print()
+            break
+        if not q:
+            continue
+        if q.lower() in ("quit", "exit", "q"):
+            break
+        ask(q)
+
+
+if __name__ == "__main__":
+    main()

config_ai.json

@@ -0,0 +1,18 @@
+{
+  "topic": "ai",
+  "topic_label": "AI",
+  "unit": "USD",
+  "source": "yfinance:NVDA",
+  "lookback": 90,
+  "horizon_steps": 7,
+  "horizon_days": 7,
+  "hidden_size": 128,
+  "num_layers": 3,
+  "dropout": 0.1,
+  "val_mae": 6.727893872130523,
+  "val_rel_mae": 0.03455442935228348,
+  "val_h1_mape": 0.018591665790424962,
+  "naive_baseline_mae": 6.687884876155594,
+  "last_date": "2026-08-14",
+  "last_value": 225.16000366210935
+}

config_economy.json

@@ -0,0 +1,18 @@
+{
+  "topic": "economy",
+  "topic_label": "Economy",
+  "unit": "USD",
+  "source": "yfinance:^GSPC",
+  "lookback": 60,
+  "horizon_steps": 7,
+  "horizon_days": 7,
+  "hidden_size": 64,
+  "num_layers": 2,
+  "dropout": 0.1,
+  "val_mae": 85.50677981465654,
+  "val_rel_mae": 0.012141019105911255,
+  "val_h1_mape": 0.00639636122128309,
+  "naive_baseline_mae": 87.09085990816496,
+  "last_date": "2026-08-14",
+  "last_value": 7785.759765625
+}

config_energy.json

@@ -0,0 +1,18 @@
+{
+  "topic": "energy",
+  "topic_label": "Energy",
+  "unit": "USD",
+  "source": "yfinance:CL=F(WTI)",
+  "lookback": 60,
+  "horizon_steps": 7,
+  "horizon_days": 7,
+  "hidden_size": 64,
+  "num_layers": 2,
+  "dropout": 0.1,
+  "val_mae": 4.7798134313437055,
+  "val_rel_mae": 0.05749673768877983,
+  "val_h1_mape": 0.02831358343785865,
+  "naive_baseline_mae": 4.782200717277816,
+  "last_date": "2026-08-18",
+  "last_value": 84.04000091552734
+}

config_finance.json

@@ -0,0 +1,18 @@
+{
+  "topic": "finance",
+  "topic_label": "Finance",
+  "unit": "USD",
+  "source": "yfinance:BTC-USD",
+  "lookback": 90,
+  "horizon_steps": 7,
+  "horizon_days": 7,
+  "hidden_size": 128,
+  "num_layers": 3,
+  "dropout": 0.1,
+  "val_mae": 1982.9512579096927,
+  "val_rel_mae": 0.028562922030687332,
+  "val_h1_mape": 0.0152207905176555,
+  "naive_baseline_mae": 1969.4407217279584,
+  "last_date": "2026-08-17",
+  "last_value": 63229.1796875
+}

config_programming.json

@@ -0,0 +1,18 @@
+{
+  "topic": "programming",
+  "topic_label": "Programming",
+  "unit": "downloads",
+  "source": "npm-api:react-daily-downloads",
+  "lookback": 90,
+  "horizon_steps": 7,
+  "horizon_days": 7,
+  "hidden_size": 128,
+  "num_layers": 3,
+  "dropout": 0.1,
+  "val_mae": 1173766.7827189027,
+  "val_rel_mae": 0.059618908911943436,
+  "val_h1_mape": 0.07118344794329942,
+  "naive_baseline_mae": 5225264.470864349,
+  "last_date": "2026-08-17",
+  "last_value": 15849863.0
+}

config_sports.json

@@ -0,0 +1,18 @@
+{
+  "topic": "sports",
+  "topic_label": "Sports",
+  "unit": "Elo",
+  "source": "huggingface:davidtadediji/tennis-atp->world#1-elo",
+  "lookback": 90,
+  "horizon_steps": 7,
+  "horizon_days": 7,
+  "hidden_size": 128,
+  "num_layers": 3,
+  "dropout": 0.1,
+  "val_mae": 3.3986784698540626,
+  "val_rel_mae": 0.0015747122233733535,
+  "val_h1_mape": 0.0005584839231664478,
+  "naive_baseline_mae": 3.1888383318009375,
+  "last_date": "2024-12-18",
+  "last_value": 2217.7739373895524
+}

config_weather_chennai_india.json

@@ -0,0 +1,18 @@
+{
+  "topic": "weather",
+  "topic_label": "Weather",
+  "unit": "K",
+  "source": "open-meteo:Chennai, India",
+  "lookback": 60,
+  "horizon_steps": 7,
+  "horizon_days": 7,
+  "hidden_size": 64,
+  "num_layers": 2,
+  "dropout": 0.1,
+  "val_mae": 0.9925612108358967,
+  "val_rel_mae": 0.0032603859435766935,
+  "val_h1_mape": 0.0018425700744697428,
+  "naive_baseline_mae": 1.0116436048493131,
+  "last_date": "2026-08-18",
+  "last_value": 303.25
+}

data.py

@@ -0,0 +1,475 @@
+import argparse
+import json
+import os
+import re
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "30")
+
+BASE_DIR = Path(__file__).resolve().parent
+DATA_DIR = BASE_DIR / "data"
+
+WEATHER_NAME = os.environ.get("WEATHER_NAME", "New York")
+WEATHER_LAT = os.environ.get("WEATHER_LAT", "40.7128")
+WEATHER_LON = os.environ.get("WEATHER_LON", "-74.0060")
+_WEATHER_NAME = WEATHER_NAME
+_WEATHER_LAT = WEATHER_LAT
+_WEATHER_LON = WEATHER_LON
+
+TOPICS = {
+    "ai": {"label": "AI", "unit": "USD", "asset": "NVIDIA stock",
+           "desc": "NVIDIA stock close price (AI market leader)"},
+    "programming": {"label": "Programming", "unit": "downloads", "asset": "react downloads",
+                    "desc": "npm downloads for 'react' (JS ecosystem demand)"},
+    "finance": {"label": "Finance", "unit": "USD", "asset": "Bitcoin",
+                "desc": "Bitcoin BTC-USD daily close"},
+    "sports": {"label": "Sports", "unit": "Elo", "asset": "ATP world #1 Elo rating",
+               "desc": "ATP world #1 Elo rating, computed from full tennis match history"},
+    "weather": {"label": "Weather", "unit": "K", "asset": f"{WEATHER_NAME} temperature",
+                "desc": f"Daily mean temperature for {WEATHER_NAME} (Open-Meteo archive, stored in Kelvin)"},
+    "economy": {"label": "Economy", "unit": "USD", "asset": "S&P 500 index",
+                "desc": "S&P 500 daily close (US market benchmark)"},
+    "energy": {"label": "Energy", "unit": "USD", "asset": "crude oil price (WTI)",
+               "desc": "WTI crude oil futures daily close"},
+}
+
+DATE_COL_RE = re.compile(r"(date|time|stamp|day|period|^dt$)", re.IGNORECASE)
+TARGET_CANDIDATES = ["close", "price", "value", "usd", "amount", "open", "target", "y"]
+
+
+def _strip_tz(parsed):
+    if getattr(parsed.dt, "tz", None) is not None:
+        parsed = parsed.dt.tz_localize(None)
+    return parsed
+
+
+def _find_date_col(df):
+    for col in df.columns:
+        if DATE_COL_RE.search(str(col)):
+            parsed = pd.to_datetime(df[col], errors="coerce")
+            if parsed.notna().mean() > 0.8:
+                return col, _strip_tz(parsed)
+    for col in df.columns:
+        if str(df[col].dtype).startswith("datetime"):
+            return col, _strip_tz(pd.to_datetime(df[col]))
+    for col in df.columns:
+        parsed = pd.to_datetime(df[col], errors="coerce")
+        if parsed.notna().mean() > 0.8:
+            return col, _strip_tz(parsed)
+    return None, None
+
+
+def _find_target_col(df, date_col):
+    numeric_cols = [c for c in df.select_dtypes(include=[np.number]).columns if c != date_col]
+    series_map = {c: df[c] for c in numeric_cols}
+    for col in df.columns:
+        if col == date_col or col in series_map:
+            continue
+        coerced = pd.to_numeric(df[col], errors="coerce")
+        if coerced.notna().mean() > 0.6:
+            series_map[col] = coerced
+    if not series_map:
+        return None, None
+    for name in TARGET_CANDIDATES:
+        for col in series_map:
+            if name in str(col).lower():
+                return col, series_map[col]
+    best = max(series_map, key=lambda c: series_map[c].notna().mean())
+    return best, series_map[best]
+
+
+def extract_series(df, min_points=300):
+    date_col, dates = _find_date_col(df)
+    if date_col is None:
+        return None
+    target_col, values = _find_target_col(df, date_col)
+    if target_col is None:
+        return None
+    out = pd.DataFrame({"date": dates, "value": pd.to_numeric(values, errors="coerce")})
+    out = out[np.isfinite(out["value"])].dropna().sort_values("date")
+    if len(out) < min_points:
+        return None
+    out = out.set_index("date")
+    gaps = out.index.to_series().diff().dropna().dt.total_seconds()
+    if len(gaps) and gaps.median() < 6 * 3600:
+        out = out.resample("1D").mean().dropna()
+    out = out[~out.index.duplicated(keep="last")].sort_index()
+    out = out.reset_index()
+    out["date"] = out["date"].dt.strftime("%Y-%m-%d")
+    if len(out) < min_points or out["value"].std() <= 0:
+        return None
+    return out
+
+
+def _hf_search_best(queries):
+    try:
+        from huggingface_hub import HfApi, hf_hub_download
+    except Exception as exc:
+        print(f"[data] huggingface_hub unavailable: {exc}")
+        return None
+    api = HfApi()
+    repo_ids = []
+    for query in queries:
+        try:
+            results = list(api.list_datasets(search=query, limit=8))
+            results.sort(key=lambda i: -(getattr(i, "downloads", 0) or 0))
+            for info in results:
+                if info.id not in repo_ids:
+                    repo_ids.append(info.id)
+        except Exception as exc:
+            print(f"[data] search failed for {query!r}: {exc}")
+    candidates = []
+    for repo in repo_ids:
+        try:
+            files = api.list_repo_files(repo, repo_type="dataset")
+        except Exception:
+            continue
+        data_files = [f for f in files if f.endswith((".csv", ".parquet")) and "readme" not in f.lower()]
+        data_files.sort(key=lambda f: 0 if f.endswith(".csv") else 1)
+        for f in data_files[:3]:
+            try:
+                local = hf_hub_download(repo, f, repo_type="dataset")
+                df = pd.read_csv(local) if f.endswith(".csv") else pd.read_parquet(local)
+                series = extract_series(df)
+                if series is not None:
+                    candidates.append((pd.Timestamp(series["date"].iloc[-1]), len(series),
+                                       f"huggingface:{repo}::{f}", series))
+            except Exception:
+                continue
+    if not candidates:
+        return None
+    candidates.sort(key=lambda c: (c[0], c[1]), reverse=True)
+    end_date, n_rows, source, series = candidates[0]
+    print(f"[data] best HF dataset: {source} (ends {end_date.date()}, {n_rows} pts; {len(candidates)} usable)")
+    return {"source": source, "series": series}
+
+
+def _yf_fetch(symbol, start):
+    import yfinance as yf
+    df = yf.download(symbol, start=start, auto_adjust=True, progress=False)
+    if df is None or df.empty:
+        raise RuntimeError(f"no data for {symbol}")
+    if isinstance(df.columns, pd.MultiIndex):
+        df.columns = df.columns.get_level_values(0)
+    df = df.reset_index().rename(columns={"Date": "date", "Close": "close"})
+    return extract_series(df)
+
+
+def fetch_finance():
+    candidates = []
+    hf = _hf_search_best(("bitcoin price", "cryptocurrency price", "stock price daily"))
+    if hf:
+        candidates.append(hf)
+    try:
+        series = _yf_fetch("BTC-USD", "2015-01-01")
+        if series is not None:
+            candidates.append({"source": "yfinance:BTC-USD", "series": series})
+    except Exception as exc:
+        print(f"[data] yfinance BTC-USD failed: {exc}")
+    return candidates
+
+
+def fetch_ai():
+    candidates = []
+    try:
+        series = _yf_fetch("NVDA", "1999-01-01")
+        if series is not None:
+            candidates.append({"source": "yfinance:NVDA", "series": series})
+    except Exception as exc:
+        print(f"[data] yfinance NVDA failed: {exc}")
+    hf = _hf_search_best(("nvidia stock price", "ai stock price", "tech stock daily"))
+    if hf:
+        candidates.append(hf)
+    return candidates
+
+
+def fetch_programming():
+    import urllib.request
+    today = pd.Timestamp.today().strftime("%Y-%m-%d")
+    url = f"https://api.npmjs.org/downloads/range/2015-01-01:{today}/react"
+    with urllib.request.urlopen(url, timeout=30) as resp:
+        payload = json.loads(resp.read().decode())
+    rows = payload.get("downloads", [])
+    series = pd.DataFrame(rows)
+    if series.empty:
+        return []
+    series["date"] = pd.to_datetime(series["day"], errors="coerce")
+    series["value"] = pd.to_numeric(series.get("downloads"), errors="coerce")
+    series = series[np.isfinite(series["value"])]
+    series = series.dropna(subset=["date"]).set_index("date")
+    series = series[~series.index.duplicated(keep="last")].sort_index()
+    series = series.resample("1D").asfreq().ffill()
+    series["value"] = series["value"].replace(0, np.nan).ffill().bfill()
+    out = series.reset_index()
+    out["date"] = out["date"].dt.strftime("%Y-%m-%d")
+    out = out[["date", "value"]]
+    if len(out) < 300 or out["value"].std() <= 0:
+        return []
+    return [{"source": "npm-api:react-daily-downloads", "series": out}]
+
+
+def _clubelo_series(club, tries=3):
+    import io
+    import time
+    import urllib.request
+    url = f"http://api.clubelo.com/{club}"
+    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (forecast-bot)"})
+    for attempt in range(tries):
+        try:
+            with urllib.request.urlopen(req, timeout=25) as resp:
+                text = resp.read().decode()
+            df = pd.read_csv(io.StringIO(text))
+            if "From" not in df.columns or "Elo" not in df.columns:
+                return None
+            s = pd.DataFrame({"date": pd.to_datetime(df["From"], errors="coerce"),
+                              "value": pd.to_numeric(df["Elo"], errors="coerce")})
+            s = s.dropna().sort_values("date")
+            s = s[s["date"] <= pd.Timestamp.today().normalize()]
+            if len(s) < 300:
+                return None
+            s = s.set_index("date")
+            s = s[~s.index.duplicated(keep="last")]
+            s = s.resample("1D").ffill().ffill().dropna()
+            out = s.reset_index()
+            out["date"] = out["date"].dt.strftime("%Y-%m-%d")
+            return out[["date", "value"]] if len(out) >= 300 else None
+        except Exception as exc:
+            print(f"[data] ClubElo {club} attempt {attempt + 1} failed: {exc}")
+            time.sleep(1.5)
+    return None
+
+
+def _atp_world1_elo():
+    try:
+        from huggingface_hub import HfApi, hf_hub_download
+    except Exception:
+        return None
+    try:
+        api = HfApi()
+        files = api.list_repo_files("davidtadediji/tennis-atp", repo_type="dataset")
+    except Exception as exc:
+        print(f"[data] ATP repo listing failed: {exc}")
+        return None
+    import re as _re
+    year_files = {}
+    for f in files:
+        m = _re.match(r"atp_matches_(\d{4})\.csv$", f)
+        if m:
+            year_files[int(m.group(1))] = f
+    if not year_files:
+        return None
+    years = sorted(y for y in year_files if y >= 1995)
+    records = []
+    for y in years:
+        try:
+            local = hf_hub_download("davidtadediji/tennis-atp", year_files[y], repo_type="dataset")
+            df = pd.read_csv(local, usecols=["tourney_date", "winner_id", "loser_id"], low_memory=False)
+            records.append(df)
+            print(f"[data] ATP {y}: {len(df)} matches")
+        except Exception:
+            print(f"[data] ATP {y}: download/parse failed")
+            continue
+    if not records:
+        return None
+    m = pd.concat(records, ignore_index=True)
+    m["date"] = pd.to_datetime(m["tourney_date"].astype(str), format="%Y%m%d", errors="coerce")
+    m["winner_id"] = pd.to_numeric(m["winner_id"], errors="coerce")
+    m["loser_id"] = pd.to_numeric(m["loser_id"], errors="coerce")
+    m = m.dropna(subset=["date", "winner_id", "loser_id"]).astype({"winner_id": "int64", "loser_id": "int64"})
+    m = m.sort_values("date")
+    print(f"[data] ATP: {len(m)} valid matches across years {years[0]}..{years[-1]}")
+
+    K, ratings = 32.0, {}
+    top_daily = []
+    for d, grp in m.groupby("date"):
+        for _, row in grp.iterrows():
+            w, l = row["winner_id"], row["loser_id"]
+            if pd.isna(w) or pd.isna(l) or w == l:
+                continue
+            rw, rl = ratings.get(w, 1500.0), ratings.get(l, 1500.0)
+            ew = 1.0 / (1.0 + 10 ** ((rl - rw) / 400.0))
+            ratings[w] = rw + K * (1 - ew)
+            ratings[l] = rl + K * (0 - (1 - ew))
+        top_daily.append((d, max(ratings.values())))
+    if len(top_daily) < 300:
+        return None
+    s = pd.DataFrame(top_daily, columns=["date", "value"]).set_index("date")
+    s = s.resample("1D").ffill().ffill().dropna()
+    s = s[s.index <= pd.Timestamp.today().normalize()]
+    out = s.reset_index()
+    out["date"] = out["date"].dt.strftime("%Y-%m-%d")
+    if len(out) < 300 or out["value"].std() <= 0:
+        return None
+    print(f"[data] sports source: ATP world #1 Elo ({len(out)} daily points, "
+          f"{years[0]}-{years[-1]})")
+    return [{"source": "huggingface:davidtadediji/tennis-atp->world#1-elo", "series": out[["date", "value"]]}]
+
+
+def fetch_sports():
+    candidates = _atp_world1_elo()
+    if candidates:
+        return candidates
+    club = None
+    for c in ["Liverpool", "RealMadrid", "BayernMunich", "Chelsea"]:
+        club = _clubelo_series(c, tries=1)
+        if club is not None:
+            return [{"source": f"clubelo:{c}", "series": club}]
+    hf = _hf_search_best(("elo rating", "chess rating history"))
+    if hf:
+        return [hf]
+    return []
+
+
+def weather_slug() -> str:
+    return re.sub(r"[^a-z0-9]+", "_", _WEATHER_NAME.lower()).strip("_")
+
+
+def set_weather_location(name: str, lat, lon):
+    global _WEATHER_NAME, _WEATHER_LAT, _WEATHER_LON, WEATHER_NAME, WEATHER_LAT, WEATHER_LON
+    _WEATHER_NAME = WEATHER_NAME = name
+    _WEATHER_LAT = WEATHER_LAT = str(lat)
+    _WEATHER_LON = WEATHER_LON = str(lon)
+    TOPICS["weather"]["asset"] = f"{name} temperature"
+    TOPICS["weather"]["desc"] = f"Daily mean temperature for {name} (Open-Meteo archive, stored in Kelvin)"
+
+
+def artifact_suffix(topic: str) -> str:
+    """Per-domain variant suffix so different instances (e.g. cities) of the
+    same topic keep separate artifacts."""
+    if topic == "weather":
+        return f"_{weather_slug()}"
+    return ""
+
+
+def fetch_weather():
+    import json as _json
+    import urllib.request
+    end = pd.Timestamp.today().strftime("%Y-%m-%d")
+    url = ("https://archive-api.open-meteo.com/v1/archive"
+           f"?latitude={_WEATHER_LAT}&longitude={_WEATHER_LON}"
+           f"&start_date=2015-01-01&end_date={end}&daily=temperature_2m_mean&timezone=auto")
+    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (forecast-bot)"})
+    try:
+        with urllib.request.urlopen(req, timeout=45) as resp:
+            payload = _json.loads(resp.read().decode())
+        daily = payload.get("daily", {})
+        dates, temps = daily.get("time", []), daily.get("temperature_2m_mean", [])
+        s = pd.DataFrame({"date": pd.to_datetime(dates, errors="coerce"),
+                          "value": pd.to_numeric(temps, errors="coerce")})
+        s = s.dropna().sort_values("date")
+        s = s[s["date"] <= pd.Timestamp.today().normalize()]
+        if len(s) < 300:
+            return []
+        s = s.set_index("date")
+        s = s[~s.index.duplicated(keep="last")]
+        s = s.resample("1D").mean().ffill().dropna()
+        s["value"] = s["value"] + 273.15
+        out = s.reset_index()
+        out["date"] = out["date"].dt.strftime("%Y-%m-%d")
+        out = out[["date", "value"]]
+        if len(out) >= 300 and out["value"].std() > 0:
+            print(f"[data] weather source: Open-Meteo {_WEATHER_NAME} ({len(out)} daily points, Kelvin)")
+            return [{"source": f"open-meteo:{_WEATHER_NAME}", "series": out}]
+    except Exception as exc:
+        print(f"[data] Open-Meteo failed: {exc}")
+    return []
+
+
+def fetch_economy():
+    try:
+        series = _yf_fetch("^GSPC", "2000-01-01")
+        if series is not None:
+            return [{"source": "yfinance:^GSPC", "series": series}]
+    except Exception as exc:
+        print(f"[data] yfinance S&P500 failed: {exc}")
+    return []
+
+
+def fetch_energy():
+    try:
+        series = _yf_fetch("CL=F", "2000-01-01")
+        if series is not None:
+            return [{"source": "yfinance:CL=F(WTI)", "series": series}]
+    except Exception as exc:
+        print(f"[data] yfinance WTI failed: {exc}")
+    return []
+
+
+FETCHERS = {
+    "ai": fetch_ai,
+    "programming": fetch_programming,
+    "finance": fetch_finance,
+    "sports": fetch_sports,
+    "weather": fetch_weather,
+    "economy": fetch_economy,
+    "energy": fetch_energy,
+}
+
+
+def generate_synthetic(topic="finance", n=1500):
+    import zlib
+    seed = zlib.crc32(topic.encode()) % (2**32)
+    rng = np.random.default_rng(seed)
+    t = np.arange(n)
+    trend = 100 * np.exp(0.0005 * t)
+    season = 8 * np.sin(2 * np.pi * t / 365.25) + 3 * np.sin(2 * np.pi * t / 30.4)
+    ar = np.zeros(n)
+    for i in range(1, n):
+        ar[i] = 0.94 * ar[i - 1] + rng.normal(0, 1.5)
+    value = trend + season + ar + rng.normal(0, 0.5, n)
+    dates = pd.date_range("2020-01-01", periods=n, freq="D")
+    return {
+        "source": f"synthetic:{topic}",
+        "series": pd.DataFrame({"date": dates.strftime("%Y-%m-%d"), "value": value}),
+    }
+
+
+def load(topic="finance", refresh=True):
+    if topic not in TOPICS:
+        raise SystemExit(f"Unknown topic {topic!r}. Choose from: {', '.join(TOPICS)}")
+    DATA_DIR.mkdir(exist_ok=True)
+    cache_csv = DATA_DIR / f"{topic}.csv"
+    cache_meta = DATA_DIR / f"{topic}.meta.json"
+    if not refresh and cache_csv.exists():
+        series = pd.read_csv(cache_csv)
+        meta = json.loads(cache_meta.read_text()) if cache_meta.exists() else {}
+        return series, meta.get("source", "cache")
+    candidates = FETCHERS[topic]()
+    if not candidates:
+        print(f"[data] online sources failed for '{topic}', generating synthetic dataset...")
+        result = generate_synthetic(topic)
+    else:
+        result = max(candidates, key=lambda r: (pd.Timestamp(r["series"]["date"].iloc[-1]), len(r["series"])))
+        print(f"[data] selected '{result['source']}' (ends {result['series']['date'].iloc[-1]}, "
+              f"{len(result['series'])} pts)")
+    series = result["series"].reset_index(drop=True)
+    series.to_csv(cache_csv, index=False)
+    gaps = pd.to_datetime(series["date"]).diff().dropna().dt.days
+    meta = {
+        "topic": topic,
+        "source": result["source"],
+        "rows": len(series),
+        "freq_days": float(gaps.median()) if len(gaps) else 1.0,
+        "start": str(series["date"].iloc[0]),
+        "end": str(series["date"].iloc[-1]),
+    }
+    cache_meta.write_text(json.dumps(meta, indent=2))
+    return series, result["source"]
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--topic", default="finance", choices=list(TOPICS))
+    parser.add_argument("--refresh", action="store_true")
+    args = parser.parse_args()
+    series, source = load(topic=args.topic, refresh=args.refresh)
+    print(f"[data] topic: {TOPICS[args.topic]['label']} ({TOPICS[args.topic]['desc']})")
+    print(f"[data] source: {source}")
+    print(f"[data] points: {len(series)}  range: {series['date'].iloc[0]} .. {series['date'].iloc[-1]}")
+
+
+if __name__ == "__main__":
+    main()

data/ai.meta.json

@@ -0,0 +1,8 @@
+{
+  "topic": "ai",
+  "source": "yfinance:NVDA",
+  "rows": 6933,
+  "freq_days": 1.0,
+  "start": "1999-01-22",
+  "end": "2026-08-14"
+}

data/economy.meta.json

@@ -0,0 +1,8 @@
+{
+  "topic": "economy",
+  "source": "yfinance:^GSPC",
+  "rows": 6694,
+  "freq_days": 1.0,
+  "start": "2000-01-03",
+  "end": "2026-08-14"
+}

data/energy.meta.json

@@ -0,0 +1,8 @@
+{
+  "topic": "energy",
+  "source": "yfinance:CL=F(WTI)",
+  "rows": 6524,
+  "freq_days": 1.0,
+  "start": "2000-08-23",
+  "end": "2026-08-18"
+}

data/finance.meta.json

@@ -0,0 +1,8 @@
+{
+  "topic": "finance",
+  "source": "yfinance:BTC-USD",
+  "rows": 4247,
+  "freq_days": 1.0,
+  "start": "2015-01-01",
+  "end": "2026-08-17"
+}

data/programming.csv

@@ -0,0 +1,548 @@
+date,value
+2025-02-17,5122863.0
+2025-02-18,5723233.0
+2025-02-19,5827613.0
+2025-02-20,5787132.0
+2025-02-21,5036029.0
+2025-02-22,1639322.0
+2025-02-23,1575462.0
+2025-02-24,5563188.0
+2025-02-25,6040485.0
+2025-02-26,5951804.0
+2025-02-27,5968319.0
+2025-02-28,5133960.0
+2025-03-01,1796871.0
+2025-03-02,1760465.0
+2025-03-03,5672966.0
+2025-03-04,5785770.0
+2025-03-05,5844434.0
+2025-03-06,5851743.0
+2025-03-07,5227560.0
+2025-03-08,1921002.0
+2025-03-09,1725033.0
+2025-03-10,5917383.0
+2025-03-11,6269431.0
+2025-03-12,6340771.0
+2025-03-13,6111425.0
+2025-03-14,5144759.0
+2025-03-15,1815727.0
+2025-03-16,1731580.0
+2025-03-17,6015919.0
+2025-03-18,6279906.0
+2025-03-19,6228094.0
+2025-03-20,6055479.0
+2025-03-21,5526374.0
+2025-03-22,1909331.0
+2025-03-23,1788948.0
+2025-03-24,6130551.0
+2025-03-25,6395177.0
+2025-03-26,6203063.0
+2025-03-27,6210284.0
+2025-03-28,5459772.0
+2025-03-29,1900124.0
+2025-03-30,1803578.0
+2025-03-31,5954186.0
+2025-04-01,6661203.0
+2025-04-02,6576843.0
+2025-04-03,6888041.0
+2025-04-04,6670857.0
+2025-04-05,3071692.0
+2025-04-06,2919997.0
+2025-04-07,6712733.0
+2025-04-08,7212617.0
+2025-04-09,6750589.0
+2025-04-10,6579603.0
+2025-04-11,6203719.0
+2025-04-12,2453142.0
+2025-04-13,2098192.0
+2025-04-14,6567023.0
+2025-04-15,6749240.0
+2025-04-16,6874334.0
+2025-04-17,6521064.0
+2025-04-18,4355495.0
+2025-04-19,2219287.0
+2025-04-20,2190840.0
+2025-04-21,4966891.0
+2025-04-22,6906119.0
+2025-04-23,7058101.0
+2025-04-24,6836432.0
+2025-04-25,6015675.0
+2025-04-26,2157259.0
+2025-04-27,2275703.0
+2025-04-28,6425006.0
+2025-04-29,6733835.0
+2025-04-30,6651039.0
+2025-05-01,4753065.0
+2025-05-02,5271981.0
+2025-05-03,2291973.0
+2025-05-04,2402833.0
+2025-05-05,6291277.0
+2025-05-06,6992360.0
+2025-05-07,6928903.0
+2025-05-08,6566142.0
+2025-05-09,5914457.0
+2025-05-10,2190796.0
+2025-05-11,2277870.0
+2025-05-12,6741178.0
+2025-05-13,7192041.0
+2025-05-14,6994702.0
+2025-05-15,6705175.0
+2025-05-16,5998028.0
+2025-05-17,2140875.0
+2025-05-18,2410617.0
+2025-05-19,6975059.0
+2025-05-20,7315953.0
+2025-05-21,7457177.0
+2025-05-22,7130478.0
+2025-05-23,6282131.0
+2025-05-24,2543909.0
+2025-05-25,2575867.0
+2025-05-26,5777006.0
+2025-05-27,7328871.0
+2025-05-28,6910371.0
+2025-05-29,6200856.0
+2025-05-30,5962090.0
+2025-05-31,2234653.0
+2025-06-01,2493303.0
+2025-06-02,7213707.0
+2025-06-03,7394293.0
+2025-06-04,7406737.0
+2025-06-05,7473882.0
+2025-06-06,6272731.0
+2025-06-07,2274985.0
+2025-06-08,2233446.0
+2025-06-09,6181351.0
+2025-06-10,7123647.0
+2025-06-11,7654167.0
+2025-06-12,7098716.0
+2025-06-13,7076444.0
+2025-06-14,3350286.0
+2025-06-15,3354579.0
+2025-06-16,7812998.0
+2025-06-17,8188061.0
+2025-06-18,8892726.0
+2025-06-19,6626144.0
+2025-06-20,7157040.0
+2025-06-21,4059454.0
+2025-06-22,3474768.0
+2025-06-23,8076868.0
+2025-06-24,8858642.0
+2025-06-25,8441191.0
+2025-06-26,7979028.0
+2025-06-27,7203347.0
+2025-06-28,3099992.0
+2025-06-29,3355923.0
+2025-06-30,7900831.0
+2025-07-01,7699048.0
+2025-07-02,7331695.0
+2025-07-03,6882827.0
+2025-07-04,5412939.0
+2025-07-05,2536907.0
+2025-07-06,2754888.0
+2025-07-07,7472686.0
+2025-07-08,7659562.0
+2025-07-09,7060450.0
+2025-07-10,6846160.0
+2025-07-11,6487668.0
+2025-07-12,2703129.0
+2025-07-13,3412034.0
+2025-07-14,7251509.0
+2025-07-15,6966115.0
+2025-07-16,7150744.0
+2025-07-17,7552013.0
+2025-07-18,6213895.0
+2025-07-19,2370471.0
+2025-07-20,2352601.0
+2025-07-21,6716667.0
+2025-07-22,7224230.0
+2025-07-23,7381941.0
+2025-07-24,7056637.0
+2025-07-25,6554474.0
+2025-07-26,2729232.0
+2025-07-27,2496018.0
+2025-07-28,8060663.0
+2025-07-29,8300974.0
+2025-07-30,8507849.0
+2025-07-31,8767706.0
+2025-08-01,7324617.0
+2025-08-02,2655386.0
+2025-08-03,2599528.0
+2025-08-04,7102206.0
+2025-08-05,7810323.0
+2025-08-06,7633348.0
+2025-08-07,7541397.0
+2025-08-08,6699454.0
+2025-08-09,2430634.0
+2025-08-10,2613698.0
+2025-08-11,7076504.0
+2025-08-12,7453458.0
+2025-08-13,7511373.0
+2025-08-14,7542787.0
+2025-08-15,6026612.0
+2025-08-16,2542689.0
+2025-08-17,2637274.0
+2025-08-18,7329811.0
+2025-08-19,7780274.0
+2025-08-20,7827592.0
+2025-08-21,7819748.0
+2025-08-22,7202314.0
+2025-08-23,2949721.0
+2025-08-24,2773012.0
+2025-08-25,7429505.0
+2025-08-26,8137538.0
+2025-08-27,7907820.0
+2025-08-28,8355298.0
+2025-08-29,9086768.0
+2025-08-30,5783505.0
+2025-08-31,5817521.0
+2025-09-01,6216186.0
+2025-09-02,7599536.0
+2025-09-03,7851907.0
+2025-09-04,8232845.0
+2025-09-05,7037271.0
+2025-09-06,2745045.0
+2025-09-07,2873510.0
+2025-09-08,7949346.0
+2025-09-09,8227013.0
+2025-09-10,8034811.0
+2025-09-11,7985133.0
+2025-09-12,7031246.0
+2025-09-13,2720266.0
+2025-09-14,2681556.0
+2025-09-15,7808362.0
+2025-09-16,8050414.0
+2025-09-17,8242375.0
+2025-09-18,7925749.0
+2025-09-19,7139170.0
+2025-09-20,2624398.0
+2025-09-21,2601704.0
+2025-09-22,7570796.0
+2025-09-23,7753950.0
+2025-09-24,7995481.0
+2025-09-25,8117734.0
+2025-09-26,7273004.0
+2025-09-27,2785336.0
+2025-09-28,2840117.0
+2025-09-29,8031686.0
+2025-09-30,8380861.0
+2025-10-01,8319264.0
+2025-10-02,7860024.0
+2025-10-03,7071082.0
+2025-10-04,2859566.0
+2025-10-05,2747277.0
+2025-10-06,7959809.0
+2025-10-07,8451066.0
+2025-10-08,8279717.0
+2025-10-09,8255370.0
+2025-10-10,7601043.0
+2025-10-11,2901823.0
+2025-10-12,2815917.0
+2025-10-13,7407202.0
+2025-10-14,8297850.0
+2025-10-15,8455422.0
+2025-10-16,8376709.0
+2025-10-17,7905867.0
+2025-10-18,7905867.0
+2025-10-19,3381955.0
+2025-10-20,7753268.0
+2025-10-21,7753268.0
+2025-10-22,8727972.0
+2025-10-23,8727406.0
+2025-10-24,9152760.0
+2025-10-25,4017426.0
+2025-10-26,3310690.0
+2025-10-27,8725497.0
+2025-10-28,8813607.0
+2025-10-29,8652906.0
+2025-10-30,9958714.0
+2025-10-31,7583571.0
+2025-11-01,3156264.0
+2025-11-02,3109120.0
+2025-11-03,8234310.0
+2025-11-04,8872492.0
+2025-11-05,8933715.0
+2025-11-06,9048161.0
+2025-11-07,8004608.0
+2025-11-08,3439566.0
+2025-11-09,3541805.0
+2025-11-10,8635729.0
+2025-11-11,8324250.0
+2025-11-12,9096063.0
+2025-11-13,9010510.0
+2025-11-14,8286903.0
+2025-11-15,3342871.0
+2025-11-16,3501054.0
+2025-11-17,8776298.0
+2025-11-18,9445039.0
+2025-11-19,9881023.0
+2025-11-20,9408539.0
+2025-11-21,8396336.0
+2025-11-22,3733751.0
+2025-11-23,3855431.0
+2025-11-24,9128701.0
+2025-11-25,9999736.0
+2025-11-26,9956991.0
+2025-11-27,8944128.0
+2025-11-28,7606329.0
+2025-11-29,4400308.0
+2025-11-30,3950705.0
+2025-12-01,9737074.0
+2025-12-02,10790084.0
+2025-12-03,10383416.0
+2025-12-04,10826479.0
+2025-12-05,9233325.0
+2025-12-06,4617855.0
+2025-12-07,5631112.0
+2025-12-08,10796822.0
+2025-12-09,11294552.0
+2025-12-10,10553249.0
+2025-12-11,10099264.0
+2025-12-12,9461298.0
+2025-12-13,4476344.0
+2025-12-14,4484467.0
+2025-12-15,9954265.0
+2025-12-16,10678981.0
+2025-12-17,10291324.0
+2025-12-18,9869981.0
+2025-12-19,8484631.0
+2025-12-20,4027950.0
+2025-12-21,3863518.0
+2025-12-22,7832259.0
+2025-12-23,7405313.0
+2025-12-24,5627595.0
+2025-12-25,3835367.0
+2025-12-26,4336405.0
+2025-12-27,4102233.0
+2025-12-28,6002253.0
+2025-12-29,6183123.0
+2025-12-30,6100306.0
+2025-12-31,5148497.0
+2026-01-01,3745740.0
+2026-01-02,5323273.0
+2026-01-03,3533028.0
+2026-01-04,3917559.0
+2026-01-05,8666257.0
+2026-01-06,8882667.0
+2026-01-07,9417727.0
+2026-01-08,9686985.0
+2026-01-09,9123144.0
+2026-01-10,4118742.0
+2026-01-11,4107130.0
+2026-01-12,9646307.0
+2026-01-13,10391770.0
+2026-01-14,12308588.0
+2026-01-15,13372696.0
+2026-01-16,11619102.0
+2026-01-17,6615301.0
+2026-01-18,6420272.0
+2026-01-19,10950431.0
+2026-01-20,10385662.0
+2026-01-21,10953716.0
+2026-01-22,13677331.0
+2026-01-23,12123483.0
+2026-01-24,6279781.0
+2026-01-25,4425172.0
+2026-01-26,10593385.0
+2026-01-27,12293999.0
+2026-01-28,12581993.0
+2026-01-29,12297553.0
+2026-01-30,11281647.0
+2026-01-31,6705711.0
+2026-02-01,6791370.0
+2026-02-02,12651309.0
+2026-02-03,15027482.0
+2026-02-04,15290699.0
+2026-02-05,15471145.0
+2026-02-06,13766106.0
+2026-02-07,7479027.0
+2026-02-08,5910028.0
+2026-02-09,11795177.0
+2026-02-10,12671902.0
+2026-02-11,12214739.0
+2026-02-12,12027144.0
+2026-02-13,10942091.0
+2026-02-14,6703068.0
+2026-02-15,8035910.0
+2026-02-16,10343044.0
+2026-02-17,13219840.0
+2026-02-18,14257485.0
+2026-02-19,15881302.0
+2026-02-20,16453405.0
+2026-02-21,9517248.0
+2026-02-22,9818293.0
+2026-02-23,15903144.0
+2026-02-24,14584573.0
+2026-02-25,14099271.0
+2026-02-26,13491902.0
+2026-02-27,12164291.0
+2026-02-28,6829622.0
+2026-03-01,9014571.0
+2026-03-02,15154755.0
+2026-03-03,13812079.0
+2026-03-04,15558446.0
+2026-03-05,14240355.0
+2026-03-06,13554436.0
+2026-03-07,9026047.0
+2026-03-08,8398710.0
+2026-03-09,15164077.0
+2026-03-10,16252647.0
+2026-03-11,15374518.0
+2026-03-12,16057587.0
+2026-03-13,15243171.0
+2026-03-14,9237662.0
+2026-03-15,7813148.0
+2026-03-16,17281411.0
+2026-03-17,18185846.0
+2026-03-18,18326323.0
+2026-03-19,16674681.0
+2026-03-20,14885848.0
+2026-03-21,10066731.0
+2026-03-22,9621109.0
+2026-03-23,17035535.0
+2026-03-24,19126493.0
+2026-03-25,21034761.0
+2026-03-26,20983183.0
+2026-03-27,19271654.0
+2026-03-28,12477272.0
+2026-03-29,11936061.0
+2026-03-30,21773918.0
+2026-03-31,21548205.0
+2026-04-01,21248268.0
+2026-04-02,18379222.0
+2026-04-03,14276457.0
+2026-04-04,9591739.0
+2026-04-05,9081539.0
+2026-04-06,14788196.0
+2026-04-07,18790739.0
+2026-04-08,18819219.0
+2026-04-09,19529827.0
+2026-04-10,18476344.0
+2026-04-11,11249697.0
+2026-04-12,10819924.0
+2026-04-13,19230270.0
+2026-04-14,20922821.0
+2026-04-15,21066037.0
+2026-04-16,20601999.0
+2026-04-17,19411762.0
+2026-04-18,11613733.0
+2026-04-19,12341280.0
+2026-04-20,20855871.0
+2026-04-21,22346566.0
+2026-04-22,22694683.0
+2026-04-23,21791368.0
+2026-04-24,19408047.0
+2026-04-25,11369236.0
+2026-04-26,11743350.0
+2026-04-27,22110773.0
+2026-04-28,21906188.0
+2026-04-29,21688858.0
+2026-04-30,20123015.0
+2026-05-01,16230866.0
+2026-05-02,10806131.0
+2026-05-03,10867887.0
+2026-05-04,19462852.0
+2026-05-05,20987259.0
+2026-05-06,21251702.0
+2026-05-07,22011461.0
+2026-05-08,19818763.0
+2026-05-09,11751379.0
+2026-05-10,11210577.0
+2026-05-11,21870894.0
+2026-05-12,23284098.0
+2026-05-13,23002779.0
+2026-05-14,21557144.0
+2026-05-15,20356013.0
+2026-05-16,11975869.0
+2026-05-17,12071258.0
+2026-05-18,22061163.0
+2026-05-19,23111057.0
+2026-05-20,23035583.0
+2026-05-21,22527385.0
+2026-05-22,20513357.0
+2026-05-23,11751361.0
+2026-05-24,12354157.0
+2026-05-25,18363758.0
+2026-05-26,21548621.0
+2026-05-27,22256826.0
+2026-05-28,22557007.0
+2026-05-29,20251890.0
+2026-05-30,11664939.0
+2026-05-31,11524819.0
+2026-06-01,22288052.0
+2026-06-02,23866895.0
+2026-06-03,23866895.0
+2026-06-04,22888438.0
+2026-06-05,20895821.0
+2026-06-06,11719888.0
+2026-06-07,11742947.0
+2026-06-08,22158642.0
+2026-06-09,23410635.0
+2026-06-10,23937751.0
+2026-06-11,24528694.0
+2026-06-12,23419933.0
+2026-06-13,12928350.0
+2026-06-14,12156421.0
+2026-06-15,23213490.0
+2026-06-16,25978689.0
+2026-06-17,25461408.0
+2026-06-18,24329678.0
+2026-06-19,21580543.0
+2026-06-20,14499232.0
+2026-06-21,14869321.0
+2026-06-22,25186034.0
+2026-06-23,24376636.0
+2026-06-24,23812876.0
+2026-06-25,23542400.0
+2026-06-26,21890754.0
+2026-06-27,13852478.0
+2026-06-28,13586133.0
+2026-06-29,23709054.0
+2026-06-30,24084253.0
+2026-07-01,24059662.0
+2026-07-02,24255468.0
+2026-07-03,20313903.0
+2026-07-04,12649383.0
+2026-07-05,12488789.0
+2026-07-06,23666271.0
+2026-07-07,26440979.0
+2026-07-08,26299185.0
+2026-07-09,25414879.0
+2026-07-10,23799330.0
+2026-07-11,15207476.0
+2026-07-12,15207476.0
+2026-07-13,26705305.0
+2026-07-14,27460609.0
+2026-07-15,25957585.0
+2026-07-16,26232443.0
+2026-07-17,24016920.0
+2026-07-18,15711694.0
+2026-07-19,15139841.0
+2026-07-20,25608881.0
+2026-07-21,27007164.0
+2026-07-22,27571725.0
+2026-07-23,26888915.0
+2026-07-24,24781398.0
+2026-07-25,14627382.0
+2026-07-26,14866416.0
+2026-07-27,26770758.0
+2026-07-28,27755816.0
+2026-07-29,26997003.0
+2026-07-30,26772295.0
+2026-07-31,23864777.0
+2026-08-01,15321268.0
+2026-08-02,15105783.0
+2026-08-03,26850156.0
+2026-08-04,28227505.0
+2026-08-05,27778101.0
+2026-08-06,23966816.0
+2026-08-07,25765917.0
+2026-08-08,15082055.0
+2026-08-09,15412640.0
+2026-08-10,27438463.0
+2026-08-11,27438463.0
+2026-08-12,28221368.0
+2026-08-13,28442032.0
+2026-08-14,28442032.0
+2026-08-15,16059353.0
+2026-08-16,15849863.0
+2026-08-17,15849863.0

data/programming.meta.json

@@ -0,0 +1,8 @@
+{
+  "topic": "programming",
+  "source": "npm-api:react-daily-downloads",
+  "rows": 547,
+  "freq_days": 1.0,
+  "start": "2025-02-17",
+  "end": "2026-08-17"
+}

data/sports.meta.json

@@ -0,0 +1,8 @@
+{
+  "topic": "sports",
+  "source": "huggingface:davidtadediji/tennis-atp->world#1-elo",
+  "rows": 10944,
+  "freq_days": 1.0,
+  "start": "1995-01-02",
+  "end": "2024-12-18"
+}

data/weather.meta.json

@@ -0,0 +1,8 @@
+{
+  "topic": "weather",
+  "source": "open-meteo:Chennai, India",
+  "rows": 4248,
+  "freq_days": 1.0,
+  "start": "2015-01-01",
+  "end": "2026-08-18"
+}

debug_logger.py

@@ -0,0 +1,34 @@
+"""Debug logging. Internal logs NEVER reach the user-facing output.
+
+Enabled only when FORECAST_DEBUG=1 (or config.debug=True). All debug output
+goes to stderr, so stdout stays clean for the final assistant response.
+"""
+
+import os
+import sys
+
+_INTERNAL_PREFIXES = (
+    "thought:", "user wants", "$ python", "executing", "running tool",
+    "model inference", "the prediction has been computed", "debug:",
+)
+
+
+def is_debug() -> bool:
+    return os.environ.get("FORECAST_DEBUG", "").lower() in ("1", "true", "yes")
+
+
+def debug(msg: str, enabled: bool = None):
+    if enabled is None:
+        enabled = is_debug()
+    if enabled:
+        print(f"[debug] {msg}", file=sys.stderr, flush=True)
+
+
+def strip_internal(text: str) -> str:
+    """Drop any line that looks like an internal execution log."""
+    out = []
+    for line in text.splitlines():
+        if any(line.strip().lower().startswith(p) for p in _INTERNAL_PREFIXES):
+            continue
+        out.append(line)
+    return "\n".join(out).strip()

model.py

@@ -0,0 +1,25 @@
+import torch
+import torch.nn as nn
+
+
+class LSTMForecaster(nn.Module):
+    def __init__(self, input_size=1, hidden_size=64, num_layers=2, dropout=0.1, horizon=7):
+        super().__init__()
+        self.horizon = horizon
+        self.lstm = nn.LSTM(
+            input_size=input_size,
+            hidden_size=hidden_size,
+            num_layers=num_layers,
+            batch_first=True,
+            dropout=dropout if num_layers > 1 else 0.0,
+        )
+        self.head = nn.Sequential(
+            nn.Linear(hidden_size, hidden_size),
+            nn.ReLU(),
+            nn.Dropout(dropout),
+            nn.Linear(hidden_size, horizon),
+        )
+
+    def forward(self, x):
+        out, _ = self.lstm(x)
+        return self.head(out[:, -1, :])

model_ai.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dee7474a03ee54ad7f3520b93cb39fe6a3839202d8e909dfc2d2908f3c8af6b9
+size 1400457

model_economy.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa6c16a49b9fe6fc324b21daca34830a8b5597227c471c8126716511b5babadc
+size 224939

model_energy.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdfc2cff1e92cda27a016f48d4ecb47077b050ba405731856b3eebc4d89dde8d
+size 224921

model_finance.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24f90b7fde4d0914d31a990195a4e99cc05cd2f6be2b3ffc7fae3c4285347d5c
+size 1400567

model_programming.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d7940f92a8cd48092c50935410229cf8ed918b5892a96a29c7095e5c17aec7a
+size 1400655

model_sports.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e521beb33a8e8e46cfe166b58dcceee967a483967e6e7042cb6375b2581fb37a
+size 1400545

model_weather_chennai_india.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c99536594bd56b91868633503b129d30d94eec7ac6eeab0e9c50569663e1b5b
+size 224939

multi.py

@@ -0,0 +1,62 @@
+import contextlib
+import io
+import json
+import sys
+from pathlib import Path
+
+import torch
+
+import data
+import debug_logger
+import prediction_engine
+import train
+
+BASE_DIR = Path(__file__).resolve().parent
+
+
+def _quiet():
+    """Silence internal stdout unless FORECAST_DEBUG=1."""
+    if debug_logger.is_debug():
+        return contextlib.nullcontext()
+    return contextlib.redirect_stdout(io.StringIO())
+
+
+def run_topic(topic, epochs=60, horizon_days=7, refresh_data=False, device="cpu"):
+    print(f"\nTOPIC: {data.TOPICS[topic]['label']} - {data.TOPICS[topic]['desc']}")
+    with _quiet():
+        series, source = data.load(topic=topic, refresh=refresh_data)
+        train.train_topic(topic, series, source, epochs=epochs,
+                          horizon_days=horizon_days, device=device)
+        pred = prediction_engine.run_prediction(topic, horizon_days=horizon_days,
+                                                device=device, debug=debug_logger.is_debug())
+    return pred
+
+
+def main():
+    topics = list(data.TOPICS)
+    if len(sys.argv) > 1 and sys.argv[1] not in ("--help",):
+        topics = [t for t in sys.argv[1:] if t in data.TOPICS] or topics
+    results = []
+    for topic in topics:
+        try:
+            results.append(run_topic(topic))
+        except Exception as exc:
+            print(f"[multi] topic '{topic}' failed: {exc}")
+
+    print(f"\n{'=' * 60}\nSUMMARY\n{'=' * 60}")
+    for r in results:
+        unit = r["unit"]
+        beat = r["naive_baseline_mae"] - r["val_mae"]
+        verdict = f"beats naive by {beat:,.1f} {unit}" if beat > 0 else f"~naive baseline ({unit})"
+        print(f"\n{r['label']} [{r['source']}]")
+        print(f"  accuracy: MAPE={r['val_h1_mape'] * 100:.2f}%  MAE={r['val_mae']:,.2f} {unit}  ({verdict})")
+        print(f"  {r['date_range_start']} -> {r['date_range_end']}  center ~{r['center_estimate']:,.2f} "
+              f"{unit}  outlook={r['direction']}")
+        print(f"  charts/report: {r['chart']} / {r['report']}")
+
+    (BASE_DIR / "results.json").write_text(json.dumps(results, indent=2))
+    print(f"\n[done] full results written to results.json; charts at prediction_<topic>.png")
+
+
+if __name__ == "__main__":
+    main()

predict.py

@@ -0,0 +1,143 @@
+import argparse
+import json
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import torch
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+import data
+from model import LSTMForecaster
+
+BASE_DIR = Path(__file__).resolve().parent
+
+
+def write_text_report(rows, cfg, topic):
+    unit = cfg.get("unit", "")
+    label = cfg.get("topic_label", topic)
+    beat = cfg["naive_baseline_mae"] - cfg["val_mae"]
+    lines = [
+        "=" * 62,
+        f"PREDICTION REPORT: {label}",
+        "=" * 62,
+        "",
+        f"Topic            : {topic}",
+        f"Dataset          : {cfg['source']}",
+        f"Model            : LSTM ({cfg['num_layers']} layers, {cfg['hidden_size']} hidden units)",
+        f"Last known value : {cfg['last_date']} = {cfg['last_value']:,.2f} {unit}",
+        "",
+        "VALIDATION (held-out test set)",
+        f"  Test MAE       : {cfg['val_mae']:,.2f} {unit}",
+        f"  Relative MAE   : {cfg['val_rel_mae'] * 100:.2f}%",
+        f"  H1 MAPE        : {cfg['val_h1_mape'] * 100:.2f}%",
+        f"  Naive baseline : {cfg['naive_baseline_mae']:,.2f} {unit}"
+        + (f"  (model beats it by {beat:,.2f} {unit})" if beat > 0 else ""),
+        "",
+        "FUTURE PREDICTIONS",
+    ]
+    for r in rows:
+        lines.append(f"  {r['date']}  ->  {r['predicted_value']:,.2f} {unit}")
+    lines += [
+        "",
+        "NOTE: forecasts are statistical estimates on a hold-out",
+        "validated model; no model can predict the future with 100%",
+        "accuracy. Treat these as central-point predictions.",
+        f"Generated: {pd.Timestamp.now().strftime('%Y-%m-%d %H:%M:%S')}",
+    ]
+    txt_path = BASE_DIR / f"prediction_{topic}{data.artifact_suffix(topic)}.txt"
+    txt_path.write_text("\n".join(lines), encoding="utf-8")
+    return txt_path
+
+
+def load_artifacts(topic, device="cpu"):
+    suffix = data.artifact_suffix(topic)
+    model_path = BASE_DIR / f"model_{topic}{suffix}.pt"
+    config_path = BASE_DIR / f"config_{topic}{suffix}.json"
+    if not model_path.exists() or not config_path.exists():
+        raise FileNotFoundError(f"Model for topic '{topic}' not trained yet. Run: python train.py --topic {topic}")
+    cfg = json.loads(config_path.read_text())
+    model = LSTMForecaster(
+        hidden_size=cfg["hidden_size"],
+        num_layers=cfg["num_layers"],
+        dropout=cfg["dropout"],
+        horizon=cfg["horizon_steps"],
+    ).to(device)
+    model.load_state_dict(torch.load(model_path, map_location=device))
+    model.eval()
+    return model, cfg
+
+
+def predict(topic, days=None, device="cpu"):
+    model, cfg = load_artifacts(topic, device)
+    series = pd.read_csv(BASE_DIR / "data" / f"{topic}.csv")
+    vals = series["value"].to_numpy(dtype=np.float64)
+    lb = cfg["lookback"]
+    freq_days = cfg.get("horizon_days", 7) / cfg["horizon_steps"]
+
+    want_days = float(days) if days else cfg["horizon_days"]
+    n_steps = int(round(want_days / freq_days)) if freq_days else cfg["horizon_steps"]
+
+    last_val = vals[-1]
+    rel = vals[-lb:] / last_val - 1.0
+    last_date = pd.Timestamp(cfg["last_date"])
+    rows = []
+    for step in range(n_steps):
+        x = torch.from_numpy(rel[-lb:].astype(np.float32)).unsqueeze(0).unsqueeze(-1).to(device)
+        with torch.no_grad():
+            pred_rel = model(x).cpu().numpy().flatten()
+        cur_val = last_val * (1.0 + pred_rel[0])
+        rel = np.concatenate([rel, pred_rel])
+        rows.append({
+            "date": (last_date + pd.Timedelta(days=freq_days * (step + 1))).strftime("%Y-%m-%d"),
+            "predicted_value": round(float(cur_val), 2),
+        })
+
+    recent = series.tail(lb)
+    history = pd.DataFrame({"date": recent["date"], "value": recent["value"], "predicted": np.nan})
+    future = pd.DataFrame({"date": [r["date"] for r in rows],
+                           "value": np.nan,
+                           "predicted": [r["predicted_value"] for r in rows]})
+    plot_df = pd.concat([history, future], ignore_index=True)
+    plot_df["d"] = pd.to_datetime(plot_df["date"])
+
+    chart_path = BASE_DIR / f"prediction_{topic}{data.artifact_suffix(topic)}.png"
+    fig, ax = plt.subplots(figsize=(11, 5))
+    ax.plot(plot_df["d"], plot_df["value"], label="actual", color="#2563eb")
+    ax.plot(plot_df["d"], plot_df["predicted"], label="model prediction", color="#dc2626", marker="o", markersize=4)
+    ax.axvline(last_date, color="gray", ls="--", lw=1)
+    unit = cfg.get("unit", "")
+    ax.set_title(f"{cfg.get('topic_label', topic)}: {cfg['source']}  "
+                 f"(val MAE={cfg['val_mae']:,.1f} {unit}, H1 MAPE={cfg['val_h1_mape'] * 100:.2f}%)")
+    ax.legend()
+    fig.autofmt_xdate()
+    fig.tight_layout()
+    fig.savefig(chart_path, dpi=130)
+    plt.close(fig)
+    txt_path = write_text_report(rows, cfg, topic)
+    return rows, cfg, chart_path, txt_path
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--topic", default="finance", choices=list(data.TOPICS))
+    parser.add_argument("--days", type=float, default=None, help="how many days ahead to predict")
+    parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
+    args = parser.parse_args()
+    rows, cfg, chart_path, txt_path = predict(args.topic, days=args.days, device=args.device)
+    unit = cfg.get("unit", "")
+    print(f"Topic        : {cfg.get('topic_label', args.topic)} - {cfg['source']}")
+    print(f"Last known   : {cfg['last_date']} = {cfg['last_value']:,.2f} {unit}")
+    print(f"Validated on : MAE={cfg['val_mae']:,.2f}, rel MAE={cfg['val_rel_mae'] * 100:.2f}%, "
+          f"H1 MAPE={cfg['val_h1_mape'] * 100:.2f}% (naive baseline MAE={cfg['naive_baseline_mae']:,.2f})")
+    print(f"\nFuture predictions:")
+    for r in rows:
+        print(f"  {r['date']}  ->  {r['predicted_value']:,.2f} {unit}")
+    print(f"\nChart saved to {chart_path}")
+    print(f"Text report saved to {txt_path}")
+
+
+if __name__ == "__main__":
+    main()

prediction_engine.py

@@ -0,0 +1,97 @@
+"""Prediction engine: runs model inference only. No formatting, no printing.
+
+Calculates scientifically meaningful, clearly separated metrics:
+- net_change_pct          : change along the forecast path (first -> last forecast)
+- forecast_range_pct      : (max - min) of the forecast path / first forecast
+- current_to_forecast_pct : change from the latest OBSERVED value to the final forecast
+- direction               : rising/falling/sideways via configurable threshold
+- primary_forecast        : the forecast value at the END of the requested horizon
+- center_estimate         : mean of the forecast path (a secondary statistic)
+
+The response_formatter presents these; this module never formats output.
+Internal logging goes through debug_logger (stderr).
+"""
+
+import contextlib
+import io
+
+import numpy as np
+
+import data
+import debug_logger
+import predict
+
+DEFAULT_DIRECTION_THRESHOLD_PCT = 0.2  # configurable; can become domain-specific
+
+
+def _pct(change: float, base: float) -> float:
+    return float(change / abs(base) * 100.0) if base != 0 else 0.0
+
+
+def run_prediction(topic: str, horizon_days: float = 7.0,
+                   device: str = "cpu", debug: bool = False,
+                   direction_threshold_pct: float = DEFAULT_DIRECTION_THRESHOLD_PCT) -> dict:
+    """Structured prediction result from the trained model. The inference
+    call's own stdout (status lines) is captured so nothing internal leaks."""
+    debug_logger.debug(f"prediction_engine: topic={topic} horizon={horizon_days}d "
+                       f"thr={direction_threshold_pct}%", debug)
+    buf = io.StringIO()
+    with contextlib.redirect_stdout(buf):
+        rows, cfg, chart_path, txt_path = predict.predict(topic, days=horizon_days, device=device)
+    debug_logger.debug(f"prediction_engine internal: {buf.getvalue()[:300]!r}", debug)
+
+    vals = np.array([r["predicted_value"] for r in rows], dtype=np.float64)
+    if len(vals) == 0 or not np.all(np.isfinite(vals)):
+        raise ValueError("model produced an empty or invalid forecast")
+
+    first = float(vals[0])
+    last = float(vals[-1])
+    last_known = cfg.get("last_value")
+
+    net_change_pct = _pct(last - first, first)
+    forecast_range_pct = _pct(vals.max() - vals.min(), first)
+    current_to_forecast_pct = (
+        _pct(last - float(last_known), float(last_known))
+        if last_known is not None and float(last_known) != 0 else None
+    )
+
+    if net_change_pct > direction_threshold_pct:
+        direction = "rising"
+    elif net_change_pct < -direction_threshold_pct:
+        direction = "falling"
+    else:
+        direction = "sideways"
+
+    result = {
+        "topic": topic,
+        "label": cfg.get("topic_label", topic),
+        "unit": cfg.get("unit", ""),
+        "asset": data.TOPICS.get(topic, {}).get("asset", cfg.get("topic_label", topic)),
+        "source": cfg["source"],
+        "date_range_start": rows[0]["date"],
+        "date_range_end": rows[-1]["date"],
+        "rows": rows,
+        "values": vals.tolist(),
+        "primary_forecast": last,
+        "center_estimate": float(np.mean(vals)),
+        "first_value": first,
+        "last_value": last,
+        "last_known_value": float(last_known) if last_known is not None else None,
+        "last_known_date": cfg.get("last_date"),
+        "net_change_pct": net_change_pct,
+        "forecast_range_pct": forecast_range_pct,
+        "current_to_forecast_pct": current_to_forecast_pct,
+        "direction": direction,
+        "direction_threshold_pct": direction_threshold_pct,
+        "val_mae": cfg["val_mae"],
+        "val_h1_mape": cfg["val_h1_mape"],
+        "naive_baseline_mae": cfg["naive_baseline_mae"],
+        "chart": str(chart_path),
+        "report": str(txt_path),
+    }
+    debug_logger.debug(
+        f"prediction_engine: {topic} end-of-horizon={last} direction={direction} "
+        f"net={net_change_pct:+.3f}% range={forecast_range_pct:.3f}% "
+        f"cur->end={current_to_forecast_pct if current_to_forecast_pct is None else format(current_to_forecast_pct, '+.3f')}%",
+        debug)
+    return result

requirements.txt

@@ -0,0 +1,8 @@
+torch>=2.0
+numpy
+pandas
+matplotlib
+scikit-learn
+datasets
+huggingface_hub
+yfinance

response.py

@@ -0,0 +1,143 @@
+"""Orchestrator: intent detection -> prediction engine -> response formatter.
+
+Only the final formatted answer is returned; internal execution details stay
+in debug_logger (stderr, off by default). Raw tool output never reaches the
+user, and each prediction is presented exactly once.
+"""
+
+import re
+import sys
+from dataclasses import dataclass, field
+from typing import Iterator, Optional
+
+import debug_logger
+import prediction_engine
+import response_formatter
+from data import TOPICS
+
+SYSTEM_PROMPT = (
+    "You are a forecasting assistant. Present the model's prediction directly "
+    "and naturally, exactly once. Never echo the user's prompt, never expose "
+    "internal execution details or reasoning, and never present a forecast "
+    "as guaranteed."
+)
+
+INTENT_PATTERNS = {
+    "finance": {"bitcoin", "btc", "crypto", "cryptocurrency"},
+    "ai": {"nvidia", "nvda", "gpu"},
+    "programming": {"npm", "react", "downloads", "javascript"},
+    "sports": {"tennis", "atp", "elo", "sport", "rating"},
+    "weather": {"weather", "rain", "raining", "temperature", "snow", "sunny",
+                "storm", "humidity", "forecast"},
+    "economy": {"economy", "gspc", "sp500", "market", "stocks", "inflation"},
+    "energy": {"oil", "wti", "crude", "energy", "gas", "petroleum"},
+}
+
+
+@dataclass
+class GenerationConfig:
+    debug: bool = False
+    streaming: bool = False
+    chunk_delay: float = 0.02
+    max_history_turns: int = 8
+    max_input_chars: int = 4000
+    default_horizon_days: float = 7.0
+    detailed: bool = False
+
+
+@dataclass
+class ConversationContext:
+    turns: list = field(default_factory=list)
+    max_turns: int = 8
+
+    def add(self, role: str, content: str):
+        self.turns.append({"role": role, "content": content})
+        if len(self.turns) > self.max_turns * 2:
+            self.turns = self.turns[-self.max_turns * 2:]
+
+
+def detect_intent(query: str, config: GenerationConfig) -> dict:
+    q = query.lower()
+    tokens = set(re.findall(r"[a-z0-9$%-]+", q))
+
+    unsupported = tokens & {"earthquake", "volcano", "traffic", "election", "lottery",
+                            "population", "pandemic"}
+    topics = [t for t in TOPICS if t in tokens or tokens & INTENT_PATTERNS.get(t, set())]
+    explicit_all = any(w in q for w in ("compare", "all topics", "everything", "each topic"))
+    greeting = bool(tokens & {"hi", "hello", "hey", "thanks"}) and len(tokens) <= 3
+
+    m = re.search(r"(\d+)\s*(?:-|\s)?\s*(?:days?|d)\b", q)
+    if m:
+        horizon = float(m.group(1))
+    elif any(w in q for w in ("month", "30 days", "30d")):
+        horizon = 30.0
+    elif "tomorrow" in tokens:
+        horizon = 1.0
+    elif "week" in tokens:
+        horizon = 7.0
+    else:
+        horizon = config.default_horizon_days
+
+    detailed = config.detailed or bool(tokens & {"detail", "detailed", "breakdown", "table", "chart"})
+    return {"greeting": greeting, "topics": topics, "compare": explicit_all,
+            "horizon": horizon, "detailed": detailed,
+            "unsupported": bool(unsupported) and not topics}
+
+
+def _run_topic(topic: str, horizon: float, device: str, config: GenerationConfig):
+    try:
+        return prediction_engine.run_prediction(topic, horizon_days=horizon,
+                                                device=device, debug=config.debug)
+    except Exception as exc:
+        debug_logger.debug(f"engine error for {topic}: {exc!r}", config.debug)
+        return None
+
+
+def generate_response(query: str, device: str = "cpu",
+                      config: Optional[GenerationConfig] = None) -> str:
+    """USER INPUT -> intent -> prediction/engine -> formatted answer."""
+    config = config or GenerationConfig()
+    debug_logger.debug(f"user: {query[:200]!r}", config.debug)
+
+    query = (query or "").strip()
+    if not query:
+        return "Please ask something - for example: *what will bitcoin do next week?*"
+    query = query[: config.max_input_chars]
+
+    try:
+        intent = detect_intent(query, config)
+    except Exception as exc:
+        debug_logger.debug(f"intent error: {exc!r}", config.debug)
+        return response_formatter.format_error()
+
+    if intent["greeting"]:
+        return response_formatter.format_greeting()
+
+    if intent.get("unsupported"):
+        return response_formatter.format_unknown()
+
+    if intent["compare"] or len(intent["topics"]) > 1 or not intent["topics"]:
+        topics = intent["topics"] if intent["topics"] and intent["compare"] else list(TOPICS)
+        preds = [_run_topic(t, intent["horizon"], device, config) for t in topics]
+        if all(p is None for p in preds):
+            return response_formatter.format_error()
+        return response_formatter.format_comparison([p for p in preds if p is not None],
+                                                    debug=config.debug)
+
+    pred = _run_topic(intent["topics"][0], intent["horizon"], device, config)
+    if pred is None:
+        return response_formatter.format_error()
+    return response_formatter.format_single(pred, detailed=intent["detailed"],
+                                            debug=config.debug)
+
+
+def stream_response(query: str, device: str = "cpu",
+                    config: Optional[GenerationConfig] = None,
+                    pregenerated: Optional[str] = None) -> Iterator[str]:
+    """Streaming: the full answer is generated first (or passed in), then
+    delivered line by line, so no partial or internal fragments reach the UI."""
+    config = config or GenerationConfig()
+    full = pregenerated if pregenerated is not None else generate_response(
+        query, device=device, config=config)
+    for chunk in full.splitlines(True):
+        yield chunk

response_formatter.py

@@ -0,0 +1,182 @@
+"""Response formatter: presents structured prediction metrics naturally.
+
+Never alters predictions, never invents confidence/probability/factors.
+Only displays metrics the prediction engine actually computed. Adapts
+wording to direction and prediction type.
+"""
+
+import pandas as pd
+
+import debug_logger
+
+_RISK_NOTES = {
+    "USD": "This is a model-generated forecast, and actual market behavior may differ.",
+    "Elo": "This is a model-generated forecast; actual results depend on future matches.",
+    "downloads": "This is a model-generated estimate; actual download counts may differ.",
+    "K": "This is a model-generated forecast; actual weather may differ.",
+}
+_DEFAULT_NOTE = "This is a model-generated estimate, and actual values may differ."
+
+
+def fmt_range_dates(start: str, end: str) -> str:
+    a, b = pd.Timestamp(start), pd.Timestamp(end)
+    if a.date() == b.date():
+        return f"{a.strftime('%B')} {a.day}, {a.year}"
+    if (a.year, a.month) == (b.year, b.month):
+        return f"{a.strftime('%B')} {a.day}-{b.day}, {a.year}"
+    if a.year == b.year:
+        return f"{a.strftime('%B')} {a.day} - {b.strftime('%B')} {b.day}, {b.year}"
+    return f"{a.strftime('%B %d, %Y')} - {b.strftime('%B %d, %Y')}"
+
+
+def fmt_value(v: float, unit: str) -> str:
+    if v is None:
+        return "n/a"
+    if unit == "downloads":
+        return f"{v / 1e6:.2f}M" if abs(v) >= 1e6 else f"{v:,.0f}"
+    if unit == "USD":
+        return f"${v:,.0f}" if abs(v) >= 1000 else f"${v:,.2f}"
+    if unit == "K":
+        return f"{v - 273.15:.1f} °C"
+    if unit:
+        return f"{v:,.2f} {unit}"
+    return f"{v:,.2f}"
+
+
+def fmt_pct(pct: float, signed: bool = False) -> str:
+    """Human-readable percent; very small values shown as <0.01%."""
+    if pct is None:
+        return "n/a"
+    if abs(pct) < 0.005:
+        return "<0.01%"
+    return f"{pct:+.2f}%" if signed else f"{pct:.2f}%"
+
+
+def _horizon_word(start: str, end: str) -> str:
+    days = (pd.Timestamp(end) - pd.Timestamp(start)).days + 1
+    if days <= 1:
+        return "tomorrow"
+    if days == 7:
+        return "next week"
+    if days == 30:
+        return "next month"
+    return f"over the next {days} days"
+
+
+_DIRECTION_VERB = {"sideways": "relatively sideways", "rising": "upward", "falling": "downward"}
+_DIRECTION_LABEL = {"sideways": "Sideways", "rising": "Rising", "falling": "Falling"}
+
+
+def _interpretation(pred: dict, unit: str, horizon_word: str) -> str:
+    asset = pred.get("asset", pred["label"])
+    direction = pred["direction"]
+    end_val = fmt_value(pred["primary_forecast"], unit)
+    start_val = fmt_value(pred["first_value"], unit)
+
+    if direction == "rising":
+        return (f"The model forecasts {_DIRECTION_VERB['rising']} movement for {asset} {horizon_word}, "
+                f"with the predicted level increasing from {start_val} to approximately **{end_val}** "
+                f"({fmt_pct(pred['net_change_pct'], signed=True)} net change).")
+    if direction == "falling":
+        return (f"The model forecasts {_DIRECTION_VERB['falling']} movement for {asset} {horizon_word}, "
+                f"with the predicted level decreasing from {start_val} to approximately **{end_val}** "
+                f"({fmt_pct(pred['net_change_pct'], signed=True)} net change).")
+    return (f"The model forecasts {_DIRECTION_VERB['sideways']} movement for {asset} {horizon_word}, "
+            f"with an estimated level around **{end_val}**.")
+
+
+def _summary_sentence(pred: dict, unit: str, horizon_word: str) -> str:
+    """A single conversational sentence restating the forecast in plain words."""
+    asset = pred.get("asset", pred["label"])
+    direction = pred["direction"]
+    end_val = fmt_value(pred["primary_forecast"], unit)
+    change = fmt_pct(pred["net_change_pct"], signed=True)
+    if direction == "sideways":
+        return (f"In short, the model expects {asset} to stay roughly stable {horizon_word}, "
+                f"hovering near {end_val}.")
+    if direction == "rising":
+        return (f"In short, the model expects {asset} to climb {change} {horizon_word}, "
+                f"reaching about {end_val}.")
+    return (f"In short, the model expects {asset} to slip {change} {horizon_word}, "
+            f"settling around {end_val}.")
+
+
+def format_single(pred: dict, detailed: bool = False, debug: bool = False) -> str:
+    """Summary sentence + title, date range, interpretation, key values, note.
+    Only shows metrics that exist and are meaningful. Single coherent answer."""
+    unit = pred["unit"]
+    asset = pred.get("asset", pred["label"])
+    direction = pred["direction"]
+    horizon_word = _horizon_word(pred["date_range_start"], pred["date_range_end"])
+    end_val = fmt_value(pred["primary_forecast"], unit)
+
+    lines = [
+        _summary_sentence(pred, unit, horizon_word),
+        "",
+        f"{asset} Forecast",
+        "",
+        fmt_range_dates(pred["date_range_start"], pred["date_range_end"]),
+        "",
+        _interpretation(pred, unit, horizon_word),
+        "",
+        f"Outlook: {_DIRECTION_LABEL[direction]}",
+        f"Predicted level ({pred['date_range_end']}): ~{end_val}",
+        f"Expected change: {fmt_pct(pred['net_change_pct'], signed=True)}",
+    ]
+
+    if pred.get("forecast_range_pct") is not None:
+        lines.append(f"Forecast range: {fmt_pct(abs(pred['forecast_range_pct']))}")
+    if pred.get("current_to_forecast_pct") is not None and pred.get("last_known_value") is not None:
+        lines.append(
+            f"From latest observed value ({fmt_value(pred['last_known_value'], unit)}, "
+            f"{pred.get('last_known_date', 'latest')}): {fmt_pct(pred['current_to_forecast_pct'], signed=True)}")
+
+    if detailed:
+        lines += ["", "| Date | Prediction |", "|---|---|"]
+        for r in pred["rows"]:
+            lines.append(f"| {r['date']} | {fmt_value(r['predicted_value'], unit)} |")
+        lines += [
+            "",
+            f"Path average: {fmt_value(pred['center_estimate'], unit)}. "
+            f"Validated accuracy on unseen data: MAPE {pred['val_h1_mape'] * 100:.2f}% "
+            f"(MAE {pred['val_mae']:,.2f} {unit}).",
+        ]
+
+    lines += ["", _RISK_NOTES.get(unit, _DEFAULT_NOTE)]
+    return debug_logger.strip_internal("\n".join(lines))
+
+
+def format_comparison(preds: list, debug: bool = False) -> str:
+    """One row per topic; shows end-of-horizon forecast + net change."""
+    lines = ["Forecast Summary", "",
+             "| Topic | Outlook | Predicted (end) | Expected change | Accuracy |",
+             "|---|---|---|---|---|"]
+    for p in preds:
+        if p is None:
+            continue
+        unit = p["unit"]
+        lines.append(
+            f"| {p['label']} | {_DIRECTION_LABEL[p['direction']]} | "
+            f"~{fmt_value(p['primary_forecast'], unit)} "
+            f"({fmt_range_dates(p['date_range_start'], p['date_range_end'])}) | "
+            f"{fmt_pct(p['net_change_pct'], signed=True)} | "
+            f"MAPE {p['val_h1_mape'] * 100:.2f}% |")
+    lines += ["", _DEFAULT_NOTE]
+    return debug_logger.strip_internal("\n".join(lines))
+
+
+def format_error(message: str = "") -> str:
+    return "I hit a problem generating that forecast. Please try again in a moment."
+
+
+def format_greeting() -> str:
+    return ("Hello! I generate forecasts from trained models. Ask about **AI**, "
+            "**Programming**, **Finance**, **Sports**, **Weather**, **Economy**, "
+            "or **Energy** - for example, *what will bitcoin do next week?*")
+
+
+def format_unknown() -> str:
+    import data
+    parts = ", ".join(f"**{info['label']}**" for info in data.TOPICS.values())
+    return (f"I can forecast these topics: {parts}. "
+            f"Try: *predict finance for 14 days* or *what will the weather do tomorrow?*")

run_all.ps1

@@ -0,0 +1,21 @@
+## Full automation: fetch data -> train -> evaluate -> predict (all topics)
+## Usage: .\run_all.ps1 [-Days 7] [-Epochs 60] [-Topics ai,programming,finance,sports] [-RefreshData]
+
+param(
+    [double]$Days = 7,
+    [int]$Epochs = 60,
+    [string[]]$Topics = @("ai", "programming", "finance", "sports"),
+    [switch]$RefreshData
+)
+
+$ErrorActionPreference = "Stop"
+$py = Join-Path $PSScriptRoot ".venv\Scripts\python.exe"
+if (-not (Test-Path $py)) { $py = "python" }
+
+if ($RefreshData) {
+    foreach ($t in $Topics) {
+        & $py "$PSScriptRoot\data.py" --topic $t --refresh
+    }
+}
+
+& $py "$PSScriptRoot\multi.py" @Topics

train.py

@@ -0,0 +1,163 @@
+import argparse
+import json
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader, TensorDataset
+
+import data
+from model import LSTMForecaster
+
+
+def build_windows(values, lookback, horizon):
+    xs, ys, bases = [], [], []
+    for i in range(lookback, len(values) - horizon):
+        base = values[i - 1]
+        if not np.isfinite(base) or base <= 0:
+            continue
+        xs.append(values[i - lookback:i] / base - 1.0)
+        ys.append(values[i:i + horizon] / base - 1.0)
+        bases.append(base)
+    return (np.array(xs, dtype=np.float32), np.array(ys, dtype=np.float32), np.array(bases, dtype=np.float64))
+
+
+def evaluate(model, loader, device):
+    model.eval()
+    preds, actuals, bases = [], [], []
+    with torch.no_grad():
+        for xb, yb, bb in loader:
+            preds.append(model(xb.to(device)).cpu().numpy())
+            actuals.append(yb.numpy())
+            bases.append(bb.numpy())
+    preds = np.concatenate(preds)
+    actuals = np.concatenate(actuals)
+    bases = np.concatenate(bases)[:, None]
+    pred_usd = bases * (1.0 + preds)
+    act_usd = bases * (1.0 + actuals)
+    mae = float(np.mean(np.abs(pred_usd - act_usd)))
+    rel_mae = float(np.mean(np.abs(preds - actuals)))
+    h1_mape = float(np.mean(np.abs((act_usd[:, 0] - pred_usd[:, 0]) / act_usd[:, 0])))
+    naive_mae = float(np.mean(np.abs(act_usd - bases)))
+    return mae, rel_mae, h1_mape, naive_mae, pred_usd, act_usd
+
+
+def train_model(series, lookback=60, horizon=7, epochs=60, batch_size=32, lr=1e-3, device="cpu", seed=0,
+                hidden_size=64, num_layers=2, dropout=0.1, patience=15):
+    torch.manual_seed(seed)
+    np.random.seed(seed)
+    vals = series["value"].to_numpy(dtype=np.float64)
+    n = len(vals)
+    n_test = min(180, max(horizon * 5, n // 5))
+
+    x_all, y_all, b_all = build_windows(vals, lookback, horizon)
+    train_count = max(lookback, (n - n_test - horizon) - lookback)
+    test_start = (n - n_test) - lookback
+    x_train, y_train = x_all[:train_count], y_all[:train_count]
+    x_test, y_test, b_test = x_all[test_start:], y_all[test_start:], b_all[test_start:]
+
+    train_ds = TensorDataset(torch.from_numpy(x_train).unsqueeze(-1), torch.from_numpy(y_train))
+    train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True)
+    test_loader = DataLoader(
+        TensorDataset(torch.from_numpy(x_test).unsqueeze(-1), torch.from_numpy(y_test), torch.from_numpy(b_test)),
+        batch_size=512,
+    )
+
+    model = LSTMForecaster(hidden_size=hidden_size, num_layers=num_layers, dropout=dropout, horizon=horizon).to(device)
+    opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-5)
+    sched = torch.optim.lr_scheduler.ReduceLROnPlateau(opt, patience=6, factor=0.5)
+    crit = nn.HuberLoss()
+
+    best_mae, best_state, patience_seen = float("inf"), None, 0
+    for epoch in range(1, epochs + 1):
+        model.train()
+        total = 0.0
+        for xb, yb in train_loader:
+            opt.zero_grad()
+            loss = crit(model(xb.to(device)), yb.to(device))
+            loss.backward()
+            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+            opt.step()
+            total += float(loss.item()) * len(xb)
+        mae, rel_mae, mape, naive, _, _ = evaluate(model, test_loader, device)
+        sched.step(mae)
+        print(f"epoch {epoch:3d}/{epochs}  loss={total / len(train_ds):.5f}  "
+              f"val_MAE=${mae:,.0f}  rel_MAE={rel_mae * 100:.2f}%  H1_MAPE={mape * 100:.2f}%")
+        if mae < best_mae - 1e-6:
+            best_mae, patience_seen = mae, 0
+            best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()}
+        else:
+            patience_seen += 1
+            if patience_seen >= patience:
+                print("[train] early stopping")
+                break
+
+    if best_state is not None:
+        model.load_state_dict(best_state)
+    mae, rel_mae, h1_mape, naive_mae, _, _ = evaluate(model, test_loader, device)
+    return model, {"mae": mae, "rel_mae": rel_mae, "h1_mape": h1_mape, "naive_mae": naive_mae,
+                   "lookback": lookback, "horizon": horizon}
+
+
+def train_topic(topic, series, source, epochs=60, horizon_days=7, device="cpu", hidden_size=64, num_layers=2,
+                dropout=0.1, lookback=60, patience=15):
+    gaps_days = float(pd.to_datetime(series["date"]).diff().dropna().dt.days.median()) if len(series) > 1 else 1.0
+    horizon_steps = max(1, int(round(horizon_days / gaps_days))) if gaps_days > 0 else horizon_days
+
+    print(f"[train] topic={topic}  dataset={source}  points={len(series)}  "
+          f"horizon={horizon_steps} steps ({horizon_days} days)")
+    model, stats = train_model(series, lookback=lookback, horizon=horizon_steps, epochs=epochs, device=device,
+                               hidden_size=hidden_size, num_layers=num_layers, dropout=dropout, patience=patience)
+
+    suffix = data.artifact_suffix(topic)
+    model_path = Path(__file__).resolve().parent / f"model_{topic}{suffix}.pt"
+    config_path = Path(__file__).resolve().parent / f"config_{topic}{suffix}.json"
+    torch.save(model.state_dict(), model_path)
+    cfg = {
+        "topic": topic,
+        "topic_label": data.TOPICS[topic]["label"],
+        "unit": data.TOPICS[topic]["unit"],
+        "source": source,
+        "lookback": stats["lookback"],
+        "horizon_steps": horizon_steps,
+        "horizon_days": horizon_days,
+        "hidden_size": hidden_size,
+        "num_layers": num_layers,
+        "dropout": dropout,
+        "val_mae": stats["mae"],
+        "val_rel_mae": stats["rel_mae"],
+        "val_h1_mape": stats["h1_mape"],
+        "naive_baseline_mae": stats["naive_mae"],
+        "last_date": str(series["date"].iloc[-1]),
+        "last_value": float(series["value"].iloc[-1]),
+    }
+    config_path.write_text(json.dumps(cfg, indent=2))
+    improvement = (stats["naive_mae"] - stats["mae"]) / stats["naive_mae"] * 100 if stats["naive_mae"] else 0.0
+    print(f"\n[result] {topic}: test MAE={stats['mae']:,.2f}  rel_MAE={stats['rel_mae'] * 100:.2f}%  "
+          f"H1 MAPE={stats['h1_mape'] * 100:.2f}%")
+    print(f"[result] naive(persistence) MAE={stats['naive_mae']:,.2f}  model improvement={improvement:+.1f}%")
+    print(f"[result] model -> {model_path.name}, config -> {config_path.name}")
+    return model, cfg
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--topic", default="finance", choices=list(data.TOPICS))
+    parser.add_argument("--refresh-data", action="store_true")
+    parser.add_argument("--epochs", type=int, default=60)
+    parser.add_argument("--horizon", type=int, default=7)
+    parser.add_argument("--hidden-size", type=int, default=64)
+    parser.add_argument("--num-layers", type=int, default=2)
+    parser.add_argument("--lookback", type=int, default=60)
+    parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
+    args = parser.parse_args()
+
+    series, source = data.load(topic=args.topic, refresh=args.refresh_data)
+    train_topic(args.topic, series, source, epochs=args.epochs, horizon_days=args.horizon, device=args.device,
+                hidden_size=args.hidden_size, num_layers=args.num_layers, lookback=args.lookback)
+
+
+if __name__ == "__main__":
+    main()

weather_cities.py

@@ -0,0 +1,40 @@
+"""Weather city registry (name, latitude, longitude). Cities are keyed by
+lowercase token; add entries to expand weather coverage without touching
+any other module."""
+
+CITIES = {
+    "chennai": {"name": "Chennai, India", "lat": 13.0827, "lon": 80.2707},
+    "mumbai": {"name": "Mumbai, India", "lat": 19.076, "lon": 72.8777},
+    "delhi": {"name": "Delhi, India", "lat": 28.6139, "lon": 77.209},
+    "bengaluru": {"name": "Bengaluru, India", "lat": 12.9716, "lon": 77.5946},
+    "bangalore": {"name": "Bengaluru, India", "lat": 12.9716, "lon": 77.5946},
+    "hyderabad": {"name": "Hyderabad, India", "lat": 17.385, "lon": 78.4867},
+    "kolkata": {"name": "Kolkata, India", "lat": 22.5726, "lon": 88.3639},
+    "pune": {"name": "Pune, India", "lat": 18.5204, "lon": 73.8567},
+    "new york": {"name": "New York, USA", "lat": 40.7128, "lon": -74.006},
+    "nyc": {"name": "New York, USA", "lat": 40.7128, "lon": -74.006},
+    "london": {"name": "London, UK", "lat": 51.5072, "lon": -0.1276},
+    "tokyo": {"name": "Tokyo, Japan", "lat": 35.6762, "lon": 139.6503},
+    "singapore": {"name": "Singapore", "lat": 1.3521, "lon": 103.8198},
+    "dubai": {"name": "Dubai, UAE", "lat": 25.2048, "lon": 55.2708},
+    "berlin": {"name": "Berlin, Germany", "lat": 52.52, "lon": 13.405},
+    "paris": {"name": "Paris, France", "lat": 48.8566, "lon": 2.3522},
+    "sydney": {"name": "Sydney, Australia", "lat": -33.8688, "lon": 151.2093},
+}
+
+
+def match_location(tokens) -> str | None:
+    """Return CITIES key present in the token set; None if no known city."""
+    for key in CITIES:
+        if key in tokens:
+            return key
+    return None
+
+
+def display_names() -> list:
+    seen, out = set(), []
+    for info in CITIES.values():
+        if info["name"] not in seen:
+            seen.add(info["name"])
+            out.append(info["name"].split(",")[0])
+    return out