import os import re import json import time import traceback from pathlib import Path from typing import Dict, Any, List, Tuple import pandas as pd import gradio as gr import papermill as pm import plotly.graph_objects as go # Optional LLM (HuggingFace Inference API) try: from huggingface_hub import InferenceClient except Exception: InferenceClient = None # ========================================================= # CONFIG # ========================================================= BASE_DIR = Path(__file__).resolve().parent NB1 = os.environ.get("NB1", "datacreation.ipynb").strip() NB2 = os.environ.get("NB2", "pythonanalysis.ipynb").strip() RUNS_DIR = BASE_DIR / "runs" ART_DIR = BASE_DIR / "artifacts" PY_FIG_DIR = ART_DIR / "py" / "figures" PY_TAB_DIR = ART_DIR / "py" / "tables" PAPERMILL_TIMEOUT = int(os.environ.get("PAPERMILL_TIMEOUT", "1800")) MAX_PREVIEW_ROWS = int(os.environ.get("MAX_FILE_PREVIEW_ROWS", "50")) MAX_LOG_CHARS = int(os.environ.get("MAX_LOG_CHARS", "8000")) HF_API_KEY = os.environ.get("HF_API_KEY", "").strip() MODEL_NAME = os.environ.get("MODEL_NAME", "deepseek-ai/DeepSeek-R1").strip() HF_PROVIDER = os.environ.get("HF_PROVIDER", "novita").strip() N8N_WEBHOOK_URL = os.environ.get("N8N_WEBHOOK_URL", "").strip() LLM_ENABLED = bool(HF_API_KEY) and InferenceClient is not None llm_client = ( InferenceClient(provider=HF_PROVIDER, api_key=HF_API_KEY) if LLM_ENABLED else None ) # ========================================================= # HELPERS # ========================================================= def ensure_dirs(): for p in [RUNS_DIR, ART_DIR, PY_FIG_DIR, PY_TAB_DIR]: p.mkdir(parents=True, exist_ok=True) def stamp(): return time.strftime("%Y%m%d-%H%M%S") def tail(text: str, n: int = MAX_LOG_CHARS) -> str: return (text or "")[-n:] def _ls(dir_path: Path, exts: Tuple[str, ...]) -> List[str]: if not dir_path.is_dir(): return [] return sorted(p.name for p in dir_path.iterdir() if p.is_file() and p.suffix.lower() in exts) def _read_csv(path: Path) -> pd.DataFrame: return pd.read_csv(path, nrows=MAX_PREVIEW_ROWS) def _read_json(path: Path): with path.open(encoding="utf-8") as f: return json.load(f) def artifacts_index() -> Dict[str, Any]: return { "python": { "figures": _ls(PY_FIG_DIR, (".png", ".jpg", ".jpeg")), "tables": _ls(PY_TAB_DIR, (".csv", ".json")), }, } # ========================================================= # PIPELINE RUNNERS # ========================================================= def run_notebook(nb_name: str) -> str: ensure_dirs() nb_in = BASE_DIR / nb_name if not nb_in.exists(): return f"ERROR: {nb_name} not found." nb_out = RUNS_DIR / f"run_{stamp()}_{nb_name}" pm.execute_notebook( input_path=str(nb_in), output_path=str(nb_out), cwd=str(BASE_DIR), log_output=True, progress_bar=False, request_save_on_cell_execute=True, execution_timeout=PAPERMILL_TIMEOUT, kernel_name="python3", ) return f"Executed {nb_name}" def run_datacreation() -> str: try: log = run_notebook(NB1) csvs = [f.name for f in BASE_DIR.glob("*.csv")] return f"OK {log}\n\nCSVs now in /app:\n" + "\n".join(f" - {c}" for c in sorted(csvs)) except Exception as e: return f"FAILED {e}\n\n{traceback.format_exc()[-2000:]}" def run_pythonanalysis() -> str: try: log = run_notebook(NB2) idx = artifacts_index() figs = idx["python"]["figures"] tabs = idx["python"]["tables"] return ( f"OK {log}\n\n" f"Figures: {', '.join(figs) or '(none)'}\n" f"Tables: {', '.join(tabs) or '(none)'}" ) except Exception as e: return f"FAILED {e}\n\n{traceback.format_exc()[-2000:]}" def run_full_pipeline() -> str: logs = [] logs.append("=" * 50) logs.append("STEP 1/2: Data Creation (web scraping + synthetic data)") logs.append("=" * 50) logs.append(run_datacreation()) logs.append("") logs.append("=" * 50) logs.append("STEP 2/2: Python Analysis (sentiment, ARIMA, dashboard)") logs.append("=" * 50) logs.append(run_pythonanalysis()) return "\n".join(logs) # ========================================================= # GALLERY LOADERS # ========================================================= def _load_all_figures() -> List[Tuple[str, str]]: """Return list of (filepath, caption) for Gallery.""" items = [] for p in sorted(PY_FIG_DIR.glob("*.png")): items.append((str(p), p.stem.replace('_', ' ').title())) return items def _load_table_safe(path: Path) -> pd.DataFrame: try: if path.suffix == ".json": obj = _read_json(path) if isinstance(obj, dict): return pd.DataFrame([obj]) return pd.DataFrame(obj) return _read_csv(path) except Exception as e: return pd.DataFrame([{"error": str(e)}]) def refresh_gallery(): """Called when user clicks Refresh on Gallery tab.""" figures = _load_all_figures() idx = artifacts_index() table_choices = list(idx["python"]["tables"]) default_df = pd.DataFrame() if table_choices: default_df = _load_table_safe(PY_TAB_DIR / table_choices[0]) return ( figures if figures else [], gr.update(choices=table_choices, value=table_choices[0] if table_choices else None), default_df, ) def on_table_select(choice: str): if not choice: return pd.DataFrame([{"hint": "Select a table above."}]) path = PY_TAB_DIR / choice if not path.exists(): return pd.DataFrame([{"error": f"File not found: {choice}"}]) return _load_table_safe(path) # ========================================================= # KPI LOADER # ========================================================= def load_kpis() -> Dict[str, Any]: for candidate in [PY_TAB_DIR / "kpis.json", PY_FIG_DIR / "kpis.json"]: if candidate.exists(): try: return _read_json(candidate) except Exception: pass return {} # ========================================================= # AI DASHBOARD -- LLM picks what to display # ========================================================= DASHBOARD_SYSTEM = """You are an AI dashboard assistant for a retail analytics app focused on e-commerce return prediction and review intelligence. The user asks questions about product reviews, return risk, customer complaints, sentiment, rating distribution, and product/category return patterns. You have access to pre-computed artifacts from a Python analysis pipeline. AVAILABLE ARTIFACTS: {artifacts_json} KPI SUMMARY: {kpis_json} YOUR JOB: 1. Answer the user's question using the KPIs and available artifacts. 2. At the END of your response, output a JSON block fenced with ```json ... ``` using this exact shape: {{"show": "figure"|"table"|"none", "scope": "python", "filename": "..."}} RULES: - If the user asks about return rate, return risk, returned products, or high-risk categories, show category_return_rate.csv if available. - If the user asks about sentiment or reviews, show sentiment_counts_sampled.csv or sentiment_distribution.png. - If the user asks about ratings, show rating_distribution.png or rating_distribution.csv. - If the user asks for dashboard overview, show df_dashboard.csv. - If no artifact is relevant, use "show": "none". - Keep the answer concise and business-focused. """ JSON_BLOCK_RE = re.compile(r"```json\s*(\{.*?\})\s*```", re.DOTALL) FALLBACK_JSON_RE = re.compile(r"\{[^{}]*\"show\"[^{}]*\}", re.DOTALL) def _parse_display_directive(text: str) -> Dict[str, str]: m = JSON_BLOCK_RE.search(text) if m: try: return json.loads(m.group(1)) except json.JSONDecodeError: pass m = FALLBACK_JSON_RE.search(text) if m: try: return json.loads(m.group(0)) except json.JSONDecodeError: pass return {"show": "none"} def _clean_response(text: str) -> str: """Strip the JSON directive block from the displayed response.""" return JSON_BLOCK_RE.sub("", text).strip() def _n8n_call(msg: str) -> Tuple[str, Dict]: """Call the student's n8n webhook and return (reply, directive).""" import requests as req try: resp = req.post(N8N_WEBHOOK_URL, json={"question": msg}, timeout=20) data = resp.json() answer = data.get("answer", "No response from n8n workflow.") chart = data.get("chart", "none") if chart and chart != "none": return answer, {"show": "figure", "chart": chart} return answer, {"show": "none"} except Exception as e: return f"n8n error: {e}. Falling back to keyword matching.", None def ai_chat(user_msg: str, history: list): """Chat function for the AI Dashboard tab.""" if not user_msg or not user_msg.strip(): return history, "", None, None idx = artifacts_index() kpis = load_kpis() # Priority: n8n webhook > HF LLM > keyword fallback if N8N_WEBHOOK_URL: reply, directive = _n8n_call(user_msg) if directive is None: reply_fb, directive = _keyword_fallback(user_msg, idx, kpis) reply += "\n\n" + reply_fb elif not LLM_ENABLED: reply, directive = _keyword_fallback(user_msg, idx, kpis) else: system = DASHBOARD_SYSTEM.format( artifacts_json=json.dumps(idx, indent=2), kpis_json=json.dumps(kpis, indent=2) if kpis else "(no KPIs yet, run the pipeline first)", ) msgs = [{"role": "system", "content": system}] for entry in (history or [])[-6:]: msgs.append(entry) msgs.append({"role": "user", "content": user_msg}) try: r = llm_client.chat_completion( model=MODEL_NAME, messages=msgs, temperature=0.3, max_tokens=600, stream=False, ) raw = ( r["choices"][0]["message"]["content"] if isinstance(r, dict) else r.choices[0].message.content ) directive = _parse_display_directive(raw) reply = _clean_response(raw) except Exception as e: reply = f"LLM error: {e}. Falling back to keyword matching." reply_fb, directive = _keyword_fallback(user_msg, idx, kpis) reply += "\n\n" + reply_fb # Resolve artifacts â build interactive Plotly charts when possible chart_out = None tab_out = None show = directive.get("show", "none") if isinstance(directive, dict) else "none" fname = directive.get("filename", "") if isinstance(directive, dict) else "" chart_name = directive.get("chart", "") if isinstance(directive, dict) else "" # Interactive chart builders keyed by name. Old aliases are included so # the app will still work if an LLM/webhook returns an older chart name. chart_builders = { "sales": build_monthly_return_chart, "returns": build_monthly_return_chart, "monthly_returns": build_monthly_return_chart, "sentiment": build_sentiment_chart, "top_sellers": build_top_return_categories_chart, "top_returns": build_top_return_categories_chart, "rating": build_rating_chart, "ratings": build_rating_chart, } if chart_name and chart_name in chart_builders: chart_out = chart_builders[chart_name]() elif show == "figure" and fname: low_fname = fname.lower() if "sentiment" in low_fname: chart_out = build_sentiment_chart() elif "rating" in low_fname: chart_out = build_rating_chart() elif "category" in low_fname or "top" in low_fname or "risk" in low_fname: chart_out = build_top_return_categories_chart() elif "return" in low_fname or "dashboard" in low_fname or "monthly" in low_fname: chart_out = build_monthly_return_chart() else: chart_out = _empty_chart(f"No interactive chart for {fname}") if show == "table" and fname: fp = PY_TAB_DIR / fname if fp.exists(): tab_out = _load_table_safe(fp) else: reply += f"\n\n*(Could not find table: {fname})*" new_history = (history or []) + [ {"role": "user", "content": user_msg}, {"role": "assistant", "content": reply}, ] return new_history, "", chart_out, tab_out def _keyword_fallback(msg: str, idx: Dict, kpis: Dict) -> Tuple[str, Dict]: """Retail return/review keyword matcher when LLM is unavailable.""" msg_lower = msg.lower() if not idx["python"]["figures"] and not idx["python"]["tables"]: return ( "No artifacts found yet. Please run the pipeline first, then come back here to explore the results.", {"show": "none"}, ) reviews_rows = kpis.get("reviews_rows", "?") returns_rows = kpis.get("returns_rows", "?") n_products = kpis.get("n_products", kpis.get("n_titles", "?")) return_rate = kpis.get("estimated_return_rate", None) if isinstance(return_rate, (int, float)): return_rate_text = f"{return_rate:.1%}" else: return_rate_text = "not available" kpi_text = ( f"Quick summary: **{reviews_rows} reviews**, **{returns_rows} return records**, " f"and **{n_products} products/classes** analyzed. Estimated return rate: **{return_rate_text}**." ) if any(w in msg_lower for w in ["return", "returned", "returns", "highest return", "return rate", "risk"]): return ( f"Here are the highest return-risk products/categories. {kpi_text}", {"show": "table", "scope": "python", "filename": "category_return_rate.csv"}, ) if any(w in msg_lower for w in ["complaint", "complaints", "problem", "issues", "review", "reviews"]): return ( f"Here is the review intelligence summary. {kpi_text}", {"show": "table", "scope": "python", "filename": "sentiment_counts_sampled.csv"}, ) if any(w in msg_lower for w in ["sentiment", "positive", "negative", "neutral"]): return ( f"Here is the sentiment breakdown from customer reviews. {kpi_text}", {"show": "figure", "chart": "sentiment"}, ) if any(w in msg_lower for w in ["rating", "ratings", "stars"]): return ( f"Here is the rating distribution. {kpi_text}", {"show": "figure", "scope": "python", "filename": "rating_distribution.png"}, ) if any(w in msg_lower for w in ["dashboard", "overview", "summary", "kpi"]): return ( f"Dashboard overview: {kpi_text}\n\nAsk me about return rates, review complaints, sentiment, ratings, or high-risk products.", {"show": "table", "scope": "python", "filename": "df_dashboard.csv"}, ) return ( f"I can help analyze e-commerce returns and review intelligence. {kpi_text}\n\n" "Try asking about: **highest return-rate categories**, **review complaints**, " "**sentiment**, **ratings**, or **dashboard overview**.", {"show": "none"}, ) # ========================================================= # KPI CARDS (BubbleBusters style) # ========================================================= def render_kpi_cards() -> str: kpis = load_kpis() if not kpis: return ( '