import re import os import nltk import random from flask_cors import CORS import numpy as np from datetime import datetime, timedelta from collections import defaultdict from flask import Flask, request, jsonify from transformers import pipeline from huggingface_hub import login from sentence_transformers import SentenceTransformer from keybert import KeyBERT from sklearn.cluster import KMeans from nltk.corpus import stopwords CORNELIA = r""" /$$$$$$ /$$$$$$ /$$$$$$$ /$$ /$$ /$$$$$$$$ /$$ /$$$$$$ /$$$$$$ /$$__ $$ /$$__ $$| $$__ $$| $$$ | $$| $$_____/| $$ |_ $$_/ /$$__ $$ | $$ \__/| $$ \ $$| $$ \ $$| $$$$| $$| $$ | $$ | $$ | $$ \ $$ | $$ | $$ | $$| $$$$$$$/| $$ $$ $$| $$$$$ | $$ | $$ | $$$$$$$$ | $$ | $$ | $$| $$__ $$| $$ $$$$| $$__/ | $$ | $$ | $$__ $$ | $$ $$| $$ | $$| $$ \ $$| $$\ $$$| $$ | $$ | $$ | $$ | $$ | $$$$$$/| $$$$$$/| $$ | $$| $$ \ $$| $$$$$$$$| $$$$$$$$ /$$$$$$| $$ | $$ \______/ \______/ |__/ |__/|__/ \__/|________/|________/|______/|__/ |__/ *#================#@@#***%%##%*+====+=====%++========++=======+*&&@@@*%%#@@@@#&%+==========&%*%+===+=====+==*@@@@@@@@@#==========*@&&%==+========% @======**%%%%*====@%%#&@@@&&@#*++*%%##%%+%#%*#%%%###%&#%#%#%#%*@&&@@@%*#&@@#%#%@*##%####%#*+@##*++****%=+%#%=#@@@@@@@@@*==+%%%**=+@@@@%===++++***% %=====*%%%%%+==%#&&@@@@@&&@@*+*****%#%%+#%%=%%%%#*#%##%##%%###%=*#+**#%%++======###%####%%%**#&#%%*+*++=*==**=#@@@@@@@@@%==+%%%%==&@@@@@+=+##%+++* *====+%%%%%+=*%&&@@@@@@&@@%++%*%%%%*%**#*++%%*%*%#%%&%##%#####++==============***#%%%%##%%*%%*%#@&&##%*+===++==%&@@@@@@@@#==*%%%+=+@@@@@@+==+*%#%% %====*%***%=*#@@@@@@@@#@#%**%%%%%*%*%##%=+%*%%*%#%%&%%#%%%%#%+==========++++===**##%%%##%%%*****&&@@@@&#%*+==+++=%#@@@@@@@%==*%%%==+@@@@@@+==*+++# %===+%%%%%%=%@@@@@@@@&@@##*#%%**%%%#&%%=++*%+*%%*%&*%#%%#%#%====+++++++++++*++==++#%%%%###%*++***%#&&&&@@@&#%*%%#*==@@@@@@@*=+**%%==%@@@@@&==**+** %+==*%%%%%%+%@@@@@@@&@&&&#*##%%%%%&#%%++*%*+*%*+*#+*%#%%%#%===++*************++==+*#%%%*%#%*==+#%%%%%@&&@@@@@@&&##%=&@@@@@@@==%%%%%==&@@@@%====**% %%+%%%%%%%*======*#@#@#@&#*###%%#&##%**#*++***+*@=+%%%*##*==++++**************++===%#%****%%+++*#&#%#%%#@&&&&@@@@@##@@@@@@@@%=+%%%%*=%@@@&=======# %**%%%%+===========%&@#@&#*##%%&%%#%*#%+++***+%@==*%**#%===+++++++++*******++++++===+#%++++*%*****#&&#&%**%@@@@@@@#&@@@@@@@@@==%%%%%==&@@&====+==+ %=#*%*==============&@#@&#*%%#&%%+%&#*=+*+***#@==+=+%##+======++++++++++++++++++++++*+#*++*+%*+*%#**#&&%%%*++&#&&&%&@@@@@@@@@%=%%%%%*=%@@@=====+=% %=&++=====+===++====%@#@@#%#&#%*%@%*+=+++*+%&&+=+==##%*@&&&*++=======+++++========***%###*+*%**++%#%*%#@&%%%#**%@&#@@@@@@@@@@#=*%%%%%=*@@@&======% %+*====*%%%*==++====%@#@&##&#%+@#*+*++=++%#&&%=+=+++=======*%%***+##=+++++=*#@+#*%*+=++*%&*+=++=+%%%%%*%#@&#%*#%**&@@@@@@@@@@@==#####=+@@@@#=====# %+====*%%##+==++===+#@&@@%%#*%@%+%%*++++%#@&%**=+%#@@&#%%%%+*+****===%+**%+===%*===+&#*@++**===+**%%#%%%*%#&&#%%#*+%@@@@@@@@@@*=#####==@@@@@+====+ %====*###%%==+++===#&&&@*%#%@&%*%%%*++%##&&#%+*#@%#==+*==#===@+=+++++++++++++*=&&@@*====#==#@@*==+%%#%##%**%&@&#%#***#@@@@@@@@*=#####+=&@@@@&===== ======%**%===+====*@@@**%%#&&%%%%%%**###&@%#%%@**=========+%=#=##=++++++++++*#=====%#%*@=====*&@&+++**%###%*%#&&#%#%###@@@@@@@*=#####*=&@@@@@*==== ====@=***===+====+&&%**%*#@@#*%##%*%###@@%%%*@=#&*=+*=+**=========++++===+++=============+**#*=*@@&*****%###%*%&@####&##@@@@@@%=#####*=&@@@@@&==== ====*======++===*&@*%%%+@&@#%%#%**%%#&@@*#%*@@*+%#==*=====++++++++++++=*=+++++++++++++++++=%#%=&@@@@@#****%##%*%&@#%#@@##@@@@@%=#####*=&@@@@@@===+ +=#+======++===*&@**#%*#&@&%*#%%%#%@@@&%#%%@@#@#=*#==**+++++++++****++=%=+++***++++++++++=+&**%#@@@@@@@@%***%%#*%&&##@@@%@@@@@%=#####+=@@@@@@@*==+ *@======++====**@+%%#*%&@@&%%%%%%#@@##%#&&@&@&&@&*+===++=+++++*****+++===+++*********+++==*+%#@&&@&#@@&&@@%**+*#%%&&@@@&&@@@@@%=#####=*@@@@@@@*==+ @============%%*%*##%*@@@@#%%%#%&@&&&&###@@&@#@@@%#&=========+*****+++=@=+++********++===+%%@&@@&@#&#@@&@&@@##*%%%#&&@@&#@@@@@*=#####=*@@@@@@@+==+ @==========+%%*&%###%%&@@&##%##@@#@@&#&&#@#&@#&&@@&&%+=====+=++****+++=&=+++++****++====#&&#@@&@#@#@@&&@@&@&@@#%%%#&@@@#&@@@@&=+####%=%@@@@@@@===+ ==+=======*##+*#*###%%&@@&###@@#@#@@&##&#@##@@###&#%%%#&&#%+=*++++++++===+++++++++=====#@@#@@#@@#@#@@@@&&@#&&&@@#%#@@@&#@@@@@%=%%*+===+&@@@@@#==== ==========%##=&%%%##%%&@@@&#@&&&@@&@@%###@##%@@@@@@@@@@&&#*==*+++==============++====*%&@#@@@&@@#@#@@@@@&@@&#@&&@@&@@&%@@@@@@*=%+========&@@@*=*== ========*%###=&##*%#%%#@@@&@&@@@&&@&&*%##&&&@@@@#####&@@&%%===++*=*#%%*%%%##%*+===+++@&&@@@#&@@&@@&@@@@@@#@@@#@&&&@&&%&@@@@@@+=*==========*@&=+%=+ ========*%###=&#&%*%#*++==+==+#@@@&@@@%%&@@@&##&@@@@@&@&%&=+#====+===============@*=%#@@&&&@@@&#@#@@@@@@&@@#@@&@@@&@@&@@@@@@#====++====++==+*=##=* +======+=+###=&&@&#*===========*&@@&&@@@###&@@@@@@&&@@#&#@*=**@====*+====++====&@=*=&@#&@@@@@@@@@&@@@@@@#@#&&&@#@@&#&@@@@@@@*====+==+**%*====*#%+% =======&#=+*==*##*==============*&@@#@@@@@@@&@&###@@&&@#&@*=%+=@@=====++=====&@%=+==@@&@@#**++=*%@@@@@@#@@&@@&@@&@@@#*@@@@@@+======+%%#%%===+##%*% ======+@@#==+%&@*=====*+*+=======*@#%@@@&@@@#&@@@@&&#&&#&@&=+*===&@*======+#@*===+=%@#=+%=======%&@@@&&@@#&@@&@@#&@@@##@@@@@&======%*##%%====%#%+% ======@@@@&&@@%=====*%%%%%%==+===*+*++@@@@@@@@&%&%%##&&#&&&*=======%@@@@@#*+======%&*====%==+%*=+#&&&@@&&@@@###@@#@@@&%@@@@@@======#++*%*+===*#**% ======@@@@@@@======*#%###%%======*%#%+=++#@@@@&#&#*#@@@#%*=====+======================**==#&=%==&@@@@&&@&&##@@@@@&&@@@%@@@@@@=======#%+======*#+%% =====*@@@@@@======#*####%%%=====+%%**%#%*=%&@@&%#@@#&%%&&%=**====*========#=*==+====%==+&&=====*&##@#%%&#@@@@@@@@&#@@@%@@@@@@====+====@======%*+%% *====&@@@@@=======%*##%++%======@&&#%%**%%++#&@#&#%&&@@#%=*%*====++=====*+==+=+==+==%#&*+*%*==%@@@@@@@@@@@@@@@@@@@#@@&&@@@@@&====++===@=====+#+**% %====@@@@@========*===*##==+===#@@@@@&#%%*#**%&@&&#&#%*==*%%*===++%+==%====*=========*%=++==*&@@@@@@@@@@@@@@@@@@@@#@@#@@@@@@%====+====%=====*=+*=& %+==+@@@@=========@*##+=======&@&&@@@@@@&#%%#&#&@&@#&&%=*%%%*===***+=======%========+=&%==+&@@@@@@@@@@@@@@@@@@@@@@#@&#@@@@@@*===++===@======++*=## %%*=*@@@*========@===========#&&@@&&@@@@@@&###&#@#&@#%=*%%%%*===+**=======+=========*+**#&@@@@@@@@@@@@@@@@@@@@@@@&&&%@@@@@@@+===++===&======*%+### %%*=%@@#==+=====&=====+====+%&@@&&@@#&@@@@@@&&@#@#@&@%=+%%%%*====*++======*=========*%=%@@@@@@@@@@@@@@@@@@@@@@@@@#&&@@@@@@@&========@======**+=+%# %%*=%@@+=***===#+=========*@&#@@@&#@@&&@@@@@@@@#@#@@#@*=*%%%*====++====+==*=========*%==&@@@@@@@@@@@@@@@@@@@@@@@&&&@@@@@@@@%=*+=====*=====*+====+# %%*=%@%=+%%%*=+@=+=======%@@@%&@@@@&&@@#@@@@@@&@@@&@#@*+=%%%*=====#===+==%====+=====***=#@@@@@@@@@@@@@@@@@@@@@@&#@@@@@@@@@@*=**====@=====#=======# %%*=%@==%%%%==@=+%%%%%==%@@@@@%&@@@@@#@@#@@@@@&@@#@@&@#*=%%%*======+================%**=*@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@%==%%*==@=====+========% %===%&=======&+========*@@@@@@&%%%%%%##@@%%%%%%@##%%@##+===========+===================+=&@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@+=======*=============== @===&@======+@========*@@@@@@@@&&&&&&&&@@&&&&&&@&&&&@&#*=================================#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*======#===============& /$$$$$$ /$$$$$$ /$$$$$$$ /$$ /$$ /$$$$$$$$ /$$ /$$$$$$ /$$$$$$ /$$__ $$ /$$__ $$| $$__ $$| $$$ | $$| $$_____/| $$ |_ $$_/ /$$__ $$ | $$ \__/| $$ \ $$| $$ \ $$| $$$$| $$| $$ | $$ | $$ | $$ \ $$ | $$ | $$ | $$| $$$$$$$/| $$ $$ $$| $$$$$ | $$ | $$ | $$$$$$$$ | $$ | $$ | $$| $$__ $$| $$ $$$$| $$__/ | $$ | $$ | $$__ $$ | $$ $$| $$ | $$| $$ \ $$| $$\ $$$| $$ | $$ | $$ | $$ | $$ | $$$$$$/| $$$$$$/| $$ | $$| $$ \ $$| $$$$$$$$| $$$$$$$$ /$$$$$$| $$ | $$ \______/ \______/ |__/ |__/|__/ \__/|________/|________/|______/|__/ |__/ """ print(CORNELIA) MIN_CLUSTER_SIZE = 3 # ── Use env var for token — never hardcode ─────────────────────────────────── _hf_token = os.environ.get("HF_TOKEN") if _hf_token: login(_hf_token) # ───────────────────────────────────────────── # CONSTANTS # ───────────────────────────────────────────── CONTRACTIONS = { r"don't": "do not", r"doesn't": "does not", r"didn't": "did not", r"can't": "cannot", r"couldn't": "could not", r"won't": "will not", r"wouldn't": "would not", r"isn't": "is not", r"aren't": "are not", r"wasn't": "was not", r"weren't": "were not", r"haven't": "have not", r"hasn't": "has not", r"hadn't": "had not", r"it's": "it is", r"that's": "that is", r"there's": "there is", r"i'm": "i am", r"i've": "i have", r"i'd": "i would", r"i'll": "i will", r"you're": "you are", r"they're": "they are", r"we're": "we are", r"he's": "he is", r"she's": "she is", r"let's": "let us", } CONTRACTION_PATTERNS = [ (re.compile(pattern, re.IGNORECASE), replacement) for pattern, replacement in CONTRACTIONS.items() ] nltk.download("stopwords", quiet=True) KEYWORD_BLOCKLIST = set(stopwords.words("english")) | { "wi", "fi", "app", "really", "like", "love", "work", "works", "great", "good", "nice", "use", "needs", "need", "feels", "feel", "makes", "make", "just", "also", "even", "much", "many", "very", } AGE_BUCKETS = [ ("18–24", 18, 24), ("25–34", 25, 34), ("35–44", 35, 44), ("45–54", 45, 54), ("55–64", 55, 64), ("65–74", 65, 74), ("75–84", 75, 84), ("85–94", 85, 94), ("95–98", 95, 98), ("99+", 99, 999), ] # ───────────────────────────────────────────── # CPU-FRIENDLY BATCH SIZE # Smaller batches reduce memory pressure on CPU # and prevent the tokenizer from stalling. # ───────────────────────────────────────────── SENTIMENT_BATCH_SIZE = 8 # Truncate at 128 tokens — comments are short, # 512 is wasteful and slows every inference call. MAX_TOKEN_LENGTH = 128 # ───────────────────────────────────────────── # HELPERS # ───────────────────────────────────────────── def expand_contractions(text: str) -> str: for pattern, replacement in CONTRACTION_PATTERNS: text = pattern.sub(replacement, text) return text def clean_keywords(kw_list: list[str]) -> list[str]: return [ kw for kw in kw_list if len(kw) >= 4 and kw.lower() not in KEYWORD_BLOCKLIST ] def run_sentiment_batch(texts: list[str]) -> list[str]: """ Run sentiment classifier over a list of texts and return a list of lowercase label strings in the same order. Single batched call — never call this inside a loop. """ if not texts: return [] results = sentiment_classifier( texts, truncation=True, max_length=MAX_TOKEN_LENGTH, batch_size=SENTIMENT_BATCH_SIZE, ) return [r["label"].lower() for r in results] def labels_to_breakdown(labels: list[str]) -> dict: """Convert a list of sentiment label strings to a percentage breakdown dict.""" counts = {"positive": 0, "neutral": 0, "negative": 0} for label in labels: if label in counts: counts[label] += 1 total = len(labels) or 1 return { "positive": round(counts["positive"] / total * 100, 1), "neutral": round(counts["neutral"] / total * 100, 1), "negative": round(counts["negative"] / total * 100, 1), } def get_sentiment_breakdown(texts: list[str]) -> dict: """Convenience wrapper — batch sentiment → percentage breakdown.""" return labels_to_breakdown(run_sentiment_batch(texts)) def cluster_and_label(texts: list[str], embeddings: np.ndarray, reduced: np.ndarray): """ Shared clustering + keyword-extraction logic. Returns (labels_array, cluster_keywords_dict). """ try: import hdbscan clusterer = hdbscan.HDBSCAN( min_cluster_size=2, min_samples=1, metric="euclidean", cluster_selection_method="leaf" ) labels = clusterer.fit_predict(reduced) except ImportError: from sklearn.preprocessing import normalize labels = KMeans(n_clusters=7, random_state=42, n_init=10).fit_predict( normalize(embeddings) ) cluster_keywords = {} for label in sorted(set(labels)): if label == -1: continue indices = [j for j, l in enumerate(labels) if l == label] cluster_docs = [texts[j] for j in indices] expanded = " ".join(expand_contractions(doc) for doc in cluster_docs) keywords = kw_model.extract_keywords( expanded, keyphrase_ngram_range=(1, 1), stop_words="english", top_n=10, use_mmr=True, diversity=0.6, ) cluster_keywords[label] = clean_keywords([kw for kw, _ in keywords])[:6] return labels, cluster_keywords def reduce_embeddings(embeddings: np.ndarray, n_components: int = 5): """ UMAP reduction with PCA fallback. On CPU, PCA is often faster and accurate enough for clustering. UMAP is used only when available and the dataset is large enough to benefit from non-linear reduction. """ try: import umap # Only bother with UMAP on larger datasets; PCA is fine for small ones if embeddings.shape[0] < 50: raise ImportError("Skipping UMAP for small dataset — using PCA") reducer = umap.UMAP( n_components=n_components, random_state=42, min_dist=0.0, metric="cosine", ) return reducer.fit_transform(embeddings) except (ImportError, Exception): from sklearn.decomposition import PCA n = min(n_components, embeddings.shape[0] - 1) return PCA(n_components=n, random_state=42).fit_transform(embeddings) # ───────────────────────────────────────────── # MODELS — loaded once at startup # ───────────────────────────────────────────── sentiment_classifier = pipeline( "sentiment-analysis", model="cardiffnlp/twitter-roberta-base-sentiment-latest", truncation=True, max_length=MAX_TOKEN_LENGTH, ) emotion_classifier = pipeline( "text-classification", model="SamLowe/roberta-base-go_emotions", top_k=None, truncation=True, max_length=MAX_TOKEN_LENGTH, ) embedder = SentenceTransformer("all-MiniLM-L6-v2") kw_model = KeyBERT(model=embedder) # ───────────────────────────────────────────── # HELPER FUNCS # ───────────────────────────────────────────── def find_optimal_k(embeddings, k_min=3, k_max=5): inertias = [] k_range = range(k_min, min(k_max + 1, len(embeddings))) for k in k_range: km = KMeans(n_clusters=k, random_state=42, n_init=10) km.fit(embeddings) inertias.append(km.inertia_) drops = [inertias[i] - inertias[i + 1] for i in range(len(inertias) - 1)] return list(k_range)[np.argmax(drops) + 1] def merge_small_clusters(cluster_ids, k, embeddings, kmeans: KMeans): counts = defaultdict(int) for c in cluster_ids: counts[c] += 1 small_clusters = {c for c, count in counts.items() if count < MIN_CLUSTER_SIZE} if not small_clusters: return cluster_ids large_cluster_ids = [c for c in range(k) if c not in small_clusters] large_centroids = kmeans.cluster_centers_[large_cluster_ids] new_ids = [] for i, c in enumerate(cluster_ids): if c in small_clusters: dists = np.linalg.norm(large_centroids - embeddings[i], axis=1) new_ids.append(large_cluster_ids[np.argmin(dists)]) else: new_ids.append(c) return new_ids # ───────────────────────────────────────────── # APP # ───────────────────────────────────────────── app = Flask(__name__) CORS(app) @app.route('/') def index(): return """ CORNELIA Backend

⚙ CORNELIA — AI Analytics Backend

This is the inference server powering CORNELIA's comment analysis pipeline. It exposes a set of NLP endpoints consumed by the Flutter app to generate real-time sentiment, emotion, topic, and demographic insights from user comments.

Models

cardiffnlp/twitter-roberta-base-sentiment-latest

Sentiment

A RoBERTa model fine-tuned on ~124M tweets for 3-class sentiment classification: positive, neutral, and negative. Used to produce sentiment distributions, sentiment over time, country-level gender breakdowns, and negative outlier scoring.

POST /roberta-base-sentiment  ·  POST /roberta-base-sentiment-SOT  ·  POST /roberta-base-sentimentCO  ·  POST /roberta-base-sentiment-SCORES

SamLowe/roberta-base-go_emotions

Emotion

A RoBERTa model fine-tuned on Google's GoEmotions dataset, capable of classifying text into 28 fine-grained emotion categories. CORNELIA uses the top 5 non-neutral emotions weighted by score to produce the emotion distribution chart.

POST /roberta-base-go

all-MiniLM-L6-v2

EmbeddingsClusteringKeywords

A lightweight sentence transformer that maps text to dense 384-dimensional vectors. Used as the backbone for topic clustering (HDBSCAN/KMeans), intertopic distance mapping (UMAP/PCA), keyword co-occurrence graph construction, emerging trend detection over time, and age-group aspect sentiment correlation.

POST /all-MiniLM-L6-v2  ·  POST /all-MiniLM-L6-v2-ETO  ·  POST /all-MiniLM-L6-v2-G  ·  POST /all-MiniLM-L6-v2-COR

KeyBERT

Keyword Extraction

Built on top of all-MiniLM-L6-v2, KeyBERT extracts the most semantically representative keywords from each topic cluster using MMR (Maximal Marginal Relevance) to balance relevance and diversity. Keywords label clusters, drive the network graph, and surface emerging issues.

Used internally across all clustering endpoints

Status

✅ All models loaded and ready.

GET /health — returns {"status": "ok"}
""", 200 # ── Health check — keeps HF Space awake and lets Flutter verify connectivity ── @app.route("/health", methods=["GET"]) def health(): return jsonify({"status": "ok"}) @app.route("/roberta-base-sentiment", methods=["POST"]) def roberta_base_sentiment(): data = request.json comments = data.get("data", []) if not comments: return jsonify({"status": "error", "message": "No comments provided"}), 400 labels = run_sentiment_batch(comments) analysis = labels_to_breakdown(labels) # Convert percentages — keep two decimals to match original contract analysis = {k: round(v, 2) for k, v in analysis.items()} return jsonify({"status": "success", "results": analysis}) @app.route("/roberta-base-go", methods=["POST"]) def roberta_base_go(): data = request.get_json() comments = data.get("data", []) if not comments: return jsonify({"status": "error", "message": "No comments provided"}), 400 results = emotion_classifier( comments, truncation=True, max_length=MAX_TOKEN_LENGTH, batch_size=SENTIMENT_BATCH_SIZE, ) if isinstance(results[0], dict): results = [results] totals = {} for comment_scores in results: for item in comment_scores: label = item["label"] if label == "neutral": continue totals[label] = totals.get(label, 0.0) + item["score"] sorted_emotions = sorted(totals.items(), key=lambda x: x[1], reverse=True)[:5] total_score = sum(score for _, score in sorted_emotions) results_json = [ {"label": label.capitalize(), "value": round(score / total_score * 100, 2)} for label, score in sorted_emotions ] return jsonify({"status": "success", "results": results_json}) @app.route("/all-MiniLM-L6-v2", methods=["POST"]) def all_MiniLM_L6_v2(): data = request.get_json() raw_docs = data.get("data", []) if not raw_docs: return jsonify({"status": "error", "message": "No comments provided"}), 400 embeddings = embedder.encode(raw_docs, show_progress_bar=False, batch_size=64) reduced = reduce_embeddings(embeddings, n_components=5) # ── 2D reduction for visualization ────────────────────────── try: import umap if len(raw_docs) >= 50: reducer_2d = umap.UMAP( n_components=2, random_state=42, min_dist=0.3, metric="cosine" ) coords_2d = reducer_2d.fit_transform(embeddings) else: raise ImportError("Small dataset — PCA fallback") except (ImportError, Exception): from sklearn.decomposition import PCA coords_2d = PCA(n_components=2, random_state=42).fit_transform(embeddings) labels, cluster_keywords = cluster_and_label(raw_docs, embeddings, reduced) x_vals = coords_2d[:, 0] y_vals = coords_2d[:, 1] x_min, x_max = x_vals.min(), x_vals.max() y_min, y_max = y_vals.min(), y_vals.max() def normalize_coord(val, vmin, vmax): if vmax == vmin: return 0.5 return 0.1 + (val - vmin) / (vmax - vmin) * 0.8 output = {} for i, label in enumerate(sorted(set(labels)), start=1): if label == -1: continue indices = [j for j, l in enumerate(labels) if l == label] cluster_docs = [raw_docs[j] for j in indices] kw_labels = cluster_keywords.get(label, []) theme_name = kw_labels[0].title() if kw_labels else f"Theme {i}" cluster_x = [normalize_coord(coords_2d[j, 0], x_min, x_max) for j in indices] cluster_y = [normalize_coord(coords_2d[j, 1], y_min, y_max) for j in indices] centroid_x = sum(cluster_x) / len(cluster_x) centroid_y = sum(cluster_y) / len(cluster_y) output[theme_name.lower()] = { "comments": len(cluster_docs), "keywords": kw_labels, "x": round(float(centroid_x), 4), "y": round(float(centroid_y), 4), } return jsonify({"status": "success", "results": output}) @app.route("/roberta-base-sentiment-SOT", methods=["POST"]) def roberta_base_sentiment_SOT(): body = request.get_json() comments = body.get("data", []) if not comments: return jsonify({"status": "error", "message": "No comments provided"}), 400 texts = [c["text"] for c in comments] dates = [c["date"] for c in comments] labels_out = run_sentiment_batch(texts) labeled = [ {"date": date, "label": label} for date, label in zip(dates, labels_out) ] return jsonify({"status": "success", "results": labeled}) @app.route("/all-MiniLM-L6-v2-ETO", methods=["POST"]) def all_MiniLM_L6_v2_ETO(): body = request.get_json() comments = body.get("data", []) if not comments: return jsonify({"status": "error", "message": "No comments provided"}), 400 texts = [c["text"] for c in comments] dates = [c["date"] for c in comments] embeddings = embedder.encode(texts, show_progress_bar=False, batch_size=64) k = find_optimal_k(embeddings) kmeans = KMeans(n_clusters=k, random_state=42, n_init=10) cluster_ids = kmeans.fit_predict(embeddings).tolist() cluster_ids = merge_small_clusters(cluster_ids, k, embeddings, kmeans) unique_ids = sorted(set(cluster_ids)) id_remap = {old: new for new, old in enumerate(unique_ids)} cluster_ids = [id_remap[c] for c in cluster_ids] k = len(unique_ids) cluster_labels = {} for cluster_id in range(k): cluster_comments = [texts[i] for i, l in enumerate(cluster_ids) if l == cluster_id] if len(cluster_comments) < 2: cluster_labels[cluster_id] = cluster_comments[0][:30] continue cleaned = [re.sub(r'[^\w\s]|[\d]', ' ', c) for c in cluster_comments] combined_text = ' '.join(expand_contractions(c) for c in cleaned) keywords = kw_model.extract_keywords( combined_text, keyphrase_ngram_range=(1, 1), stop_words='english', use_mmr=True, diversity=0.5, top_n=1, ) cluster_labels[cluster_id] = keywords[0][0] if keywords else f"theme_{cluster_id}" counts = defaultdict(lambda: defaultdict(int)) for i, date_str in enumerate(dates): keyword = cluster_labels[cluster_ids[i]] counts[keyword][date_str] += 1 trend_data = [] for keyword, date_counts in counts.items(): for date, count in sorted(date_counts.items()): trend_data.append({ "time": date, "count": count, "keyword": keyword, }) return jsonify({"status": "success", "results": trend_data}) @app.route("/roberta-base-sentimentCO", methods=["POST"]) def roberta_base_sentiment_CO(): data = request.json comments = data.get("data", []) if not comments: return jsonify({"status": "error", "message": "No comments provided"}), 400 # ── Single batch call for ALL texts ───────────────────────── all_texts = [item["text"] for item in comments] sent_labels = run_sentiment_batch(all_texts) # ── Attach pre-computed label back to each item ────────────── for item, label in zip(comments, sent_labels): item["_sentiment"] = label # ── Group by country ───────────────────────────────────────── country_groups = defaultdict(list) for item in comments: country_groups[item["country"]].append(item) def sentiment_breakdown_from_items(items_subset): return labels_to_breakdown([i["_sentiment"] for i in items_subset]) output = [] for country, items in country_groups.items(): male_items = [i for i in items if i["gender"].lower() == "male"] female_items = [i for i in items if i["gender"].lower() == "female"] output.append({ "country": country, "commentsNo": len(items), "maleSentiment": sentiment_breakdown_from_items(male_items), "femaleSentiment": sentiment_breakdown_from_items(female_items), }) return jsonify({"status": "success", "results": output}) @app.route("/all-MiniLM-L6-v2-G", methods=["POST"]) def all_MiniLM_L6_v2_G(): data = request.get_json() raw_docs = data.get("data", []) if not raw_docs: return jsonify({"status": "error", "message": "No comments provided"}), 400 embeddings = embedder.encode(raw_docs, show_progress_bar=False, batch_size=64) reduced = reduce_embeddings(embeddings, n_components=5) labels, cluster_keywords = cluster_and_label(raw_docs, embeddings, reduced) all_keywords_per_doc = [ cluster_keywords.get(labels[i], []) for i in range(len(raw_docs)) ] node_counts = defaultdict(int) edge_counts = defaultdict(int) for kws in all_keywords_per_doc: unique_kws = list(set(kws)) for kw in unique_kws: node_counts[kw] += 1 for i in range(len(unique_kws)): for j in range(i + 1, len(unique_kws)): pair = tuple(sorted([unique_kws[i], unique_kws[j]])) edge_counts[pair] += 1 top_n_nodes = 16 top_keywords = sorted(node_counts.items(), key=lambda x: x[1], reverse=True)[:top_n_nodes] top_kw_set = {kw for kw, _ in top_keywords} min_cooccurrence = 1 edges = [] max_count = max(edge_counts.values()) if edge_counts else 1 for (kw1, kw2), count in edge_counts.items(): if kw1 not in top_kw_set or kw2 not in top_kw_set: continue if count < min_cooccurrence: continue edges.append({ "source": kw1, "target": kw2, "relation": "co-occurs with", "weight": round(count / max_count, 4), }) max_node_count = top_keywords[0][1] if top_keywords else 1 connected_kws = {e["source"] for e in edges} | {e["target"] for e in edges} nodes = [ { "id": kw, "label": kw, "frequency": count, "weight": round(count / max_node_count, 4), } for kw, count in top_keywords if kw in connected_kws ] return jsonify({"status": "success", "results": {"nodes": nodes, "edges": edges}}) @app.route("/all-MiniLM-L6-v2-COR", methods=["POST"]) def all_MiniLM_L6_v2_COR(): body = request.get_json() comments = body.get("data", []) if not comments: return jsonify({"status": "error", "message": "No comments provided"}), 400 def safe_int(val): try: return int(val) except (ValueError, TypeError): return None comments = [{**c, "age": safe_int(c.get("age"))} for c in comments] all_texts = [c["text"] for c in comments] # ── Encode ALL texts once ──────────────────────────────────── all_embeddings = embedder.encode(all_texts, show_progress_bar=False, batch_size=64) # ── Batch ALL sentiment in a single call ───────────────────── # This is the key optimization: one classifier call for the entire # dataset up front. All per-bucket and per-aspect sentiment work # below uses these pre-computed labels — no further classifier calls. all_sent_labels = run_sentiment_batch(all_texts) # ── Bucket by age ──────────────────────────────────────────── buckets = {label: [] for label, _, _ in AGE_BUCKETS} bucket_emb = {label: [] for label, _, _ in AGE_BUCKETS} bucket_sent = {label: [] for label, _, _ in AGE_BUCKETS} for idx, c in enumerate(comments): age = c["age"] if age is None: continue for label, lo, hi in AGE_BUCKETS: if lo <= age <= hi: buckets[label].append(c["text"]) bucket_emb[label].append(all_embeddings[idx]) bucket_sent[label].append(all_sent_labels[idx]) break results = [] for label, _, _ in AGE_BUCKETS: texts = buckets[label] if not texts: continue embeddings = np.array(bucket_emb[label]) s_labels = bucket_sent[label] # ── Sentiment from pre-computed labels (no new inference) ─ counts = {"positive": 0, "neutral": 0, "negative": 0} for sl in s_labels: counts[sl] += 1 total = len(texts) sentiment = {k: round(v / total * 100, 1) for k, v in counts.items()} dominant = max(counts, key=counts.get) # ── Reduce + cluster ────────────────────────────────────── n_comp = min(5, len(texts) - 1) try: import umap if len(texts) >= 50: reducer = umap.UMAP( n_components=n_comp, random_state=42, min_dist=0.0, metric="cosine", ) reduced = reducer.fit_transform(embeddings) else: raise ImportError("Small bucket — PCA fallback") except (ImportError, Exception): from sklearn.decomposition import PCA n = min(n_comp, len(texts) - 1) reduced = PCA(n_components=n, random_state=42).fit_transform(embeddings) try: import hdbscan clusterer = hdbscan.HDBSCAN( min_cluster_size=2, min_samples=1, metric="euclidean", cluster_selection_method="leaf", ) cluster_labels = clusterer.fit_predict(reduced) except (ImportError, Exception): from sklearn.preprocessing import normalize k = min(3, len(texts)) cluster_labels = KMeans( n_clusters=k, random_state=42, n_init=10 ).fit_predict(normalize(embeddings)) # ── Build a text→pre-computed-label lookup for this bucket ─ # We match by position: texts[i] corresponds to s_labels[i]. text_to_sent = {texts[i]: s_labels[i] for i in range(len(texts))} all_aspects = [] for cluster_id in sorted(set(cluster_labels)): if cluster_id == -1: continue indices = [j for j, l in enumerate(cluster_labels) if l == cluster_id] cluster_texts = [texts[j] for j in indices] expanded = " ".join(expand_contractions(t) for t in cluster_texts) keywords = kw_model.extract_keywords( expanded, keyphrase_ngram_range=(1, 2), stop_words="english", top_n=10, use_mmr=True, diversity=0.6, ) cleaned = clean_keywords([kw for kw, _ in keywords])[:3] for aspect in cleaned: related = [t for t in cluster_texts if aspect.lower() in t.lower()] if not related: related = cluster_texts # ── Use pre-computed labels — zero new inference ─── a_counts = {"positive": 0, "neutral": 0, "negative": 0} for t in related: lbl = text_to_sent.get(t, "neutral") if lbl in a_counts: a_counts[lbl] += 1 a_total = len(related) all_aspects.append({ "aspect": aspect, "sentiment": max(a_counts, key=a_counts.get), "breakdown": { k: round(v / a_total * 100, 1) for k, v in a_counts.items() }, "mentionCount": len(related), }) seen = {} for a in all_aspects: key = a["aspect"].lower() if key not in seen or a["mentionCount"] > seen[key]["mentionCount"]: seen[key] = a deduped_aspects = list(seen.values())[:5] results.append({ "ageGroup": label, "commentCount": total, "dominantSentiment": dominant, "sentimentBreakdown": sentiment, "aspects": deduped_aspects, }) return jsonify({"status": "success", "results": results}) @app.route("/roberta-base-sentiment-SCORES", methods=["POST"]) def roberta_base_sentiment_scores(): data = request.json comments = data.get("data", []) if not comments: return jsonify({"status": "error", "message": "No comments provided"}), 400 # top_k=None returns all label scores per text results = sentiment_classifier( comments, top_k=None, truncation=True, max_length=MAX_TOKEN_LENGTH, batch_size=SENTIMENT_BATCH_SIZE, ) output = [] for i, comment_scores in enumerate(results): neg_score = next( (s["score"] for s in comment_scores if s["label"].lower() == "negative"), 0.0, ) output.append({ "id": i, "text": comments[i], "score": round(neg_score, 4), }) return jsonify({"status": "success", "results": output}) if __name__ == "__main__": app.run(host="0.0.0.0", port=5000, debug=False)