| |
| """ |
| Phase 1.2: Base Dataset Fast-Path ChitChat Mapping & Hinglish ChitChat Generation |
| """ |
|
|
| import json |
| import os |
| import random |
| import requests |
|
|
| BASE_DIR = "/opt/vox/query-classification-dataset" |
| OUTPUT_DIR = "/opt/vox/sandbox/datasets" |
| OLLAMA_URL = "http://localhost:11434/api/generate" |
| LMS_URL = "http://localhost:1234/v1/chat/completions" |
|
|
| os.makedirs(OUTPUT_DIR, exist_ok=True) |
|
|
| def load_base_file(filename, scope, lang, source): |
| filepath = os.path.join(BASE_DIR, filename) |
| samples = [] |
| with open(filepath, "r", encoding="utf-8") as f: |
| for line in f: |
| line = line.strip() |
| if not line: |
| continue |
| data = json.loads(line) |
| text = data.get("text", "").strip() |
| if text: |
| samples.append({ |
| "text": text, |
| "scope": scope, |
| "language": lang, |
| "source": source |
| }) |
| return samples |
|
|
| def generate_hinglish_chitchat_llm(target_count=1000): |
| print(f"Generating {target_count} Hinglish ChitChat/Banter queries via local LLM...") |
| hinglish_samples = set() |
| |
| |
| prompt = """Generate 50 diverse, casual Hinglish (code-switched Hindi-English) chitchat, filler, greetings, and banter queries spoken to a voice AI assistant. |
| Examples: |
| - Hey Vox, kya haal hai? |
| - Suno yaar, good morning! |
| - Kaun ho tum, apne baare me batao? |
| - Aaj ka mausam kaisa hai bhai? |
| - Kya chal raha hai aajkal? |
| - Bhai ek joke sunao na please. |
| - Vox, tu mast hai yaar. |
| - Good night, kal milte hain. |
| - Bol na kuch, bore ho raha hoon. |
| - Kya tum mere dost banoge? |
| |
| Rules: |
| 1. Every query must be casual chitchat, greeting, banter, or filler (0 technical/code questions, 0 personal data retrieval). |
| 2. Code-switched Latin Hinglish script only (e.g. 'kya haal hai', 'kya chal raha hai', 'good morning bhai'). |
| 3. Output ONLY a raw JSON array of strings: ["query 1", "query 2", ...] |
| """ |
|
|
| attempts = 0 |
| while len(hinglish_samples) < target_count and attempts < 30: |
| attempts += 1 |
| print(f" [Attempt {attempts}] Current Hinglish ChitChat count: {len(hinglish_samples)}/{target_count}") |
| |
| |
| try: |
| payload = { |
| "model": "llama3.1:8b", |
| "prompt": prompt, |
| "stream": False, |
| "options": {"temperature": 0.85, "top_p": 0.95} |
| } |
| res = requests.post(OLLAMA_URL, json=payload, timeout=60) |
| if res.status_code == 200: |
| resp_text = res.json().get("response", "").strip() |
| |
| start_idx = resp_text.find("[") |
| end_idx = resp_text.rfind("]") |
| if start_idx != -1 and end_idx != -1: |
| raw_json = resp_text[start_idx:end_idx+1] |
| items = json.loads(raw_json) |
| for item in items: |
| item_clean = item.strip() |
| if item_clean and len(item_clean) > 3: |
| hinglish_samples.add(item_clean) |
| except Exception as e: |
| print(f" Ollama generation warning: {e}") |
| |
| |
| if len(hinglish_samples) < target_count: |
| print(f" Expanding via templates to reach {target_count}...") |
| greetings = ["hey", "hello", "hi", "suno", "bhai", "vox", "arrey", "namaste"] |
| phrases = [ |
| "kya haal hai", "kya chal raha hai", "kaise ho", "kya chal raha h", "sab badiya", |
| "good morning", "good evening", "good night", "kya bolte ho", "bore ho raha hoon", |
| "kuch batao na", "ek joke sunao", "kaise ho yaar", "tu mast hai", "kya scene hai", |
| "kaise chal raha hai sab", "kya chal rha hai", "tu kya kar raha hai", "kuch bolo na", |
| "main theek hoon tum batao", "chalo bye", "phir milte hain", "kya khabar hai" |
| ] |
| suffixes = ["bhai", "yaar", "vox", "dost", "ji", "buddy", "man", "bro"] |
| |
| while len(hinglish_samples) < target_count: |
| g = random.choice(greetings) |
| p = random.choice(phrases) |
| s = random.choice(suffixes) |
| comb = f"{g} {p} {s}".title() if random.random() < 0.2 else f"{g} {p} {s}" |
| hinglish_samples.add(comb) |
| |
| result_list = [{ |
| "text": q, |
| "scope": "ChitChat", |
| "language": "hinglish", |
| "source": "synthetic_hinglish_chitchat" |
| } for q in list(hinglish_samples)[:target_count]] |
| |
| return result_list |
|
|
| def main(): |
| print("=== Phase 1.2: Base Dataset Audit & Fast-Path ChitChat Mapping ===") |
| |
| |
| en_generic = load_base_file("en_generic.jsonl", "ChitChat", "en", "base_generic") |
| hi_generic = load_base_file("hi_generic.jsonl", "ChitChat", "hi", "base_generic") |
| print(f"Loaded {len(en_generic)} EN generic items -> ChitChat") |
| print(f"Loaded {len(hi_generic)} HI generic items -> ChitChat") |
| |
| |
| hinglish_chitchat = generate_hinglish_chitchat_llm(1000) |
| print(f"Generated {len(hinglish_chitchat)} Hinglish ChitChat items") |
| |
| all_chitchat = en_generic + hi_generic + hinglish_chitchat |
| print(f"Total ChitChat dataset count: {len(all_chitchat)}") |
| |
| |
| chitchat_file = os.path.join(OUTPUT_DIR, "chitchat_base.jsonl") |
| with open(chitchat_file, "w", encoding="utf-8") as f: |
| for item in all_chitchat: |
| f.write(json.dumps(item, ensure_ascii=False) + "\n") |
| print(f"Saved {len(all_chitchat)} ChitChat items to {chitchat_file}") |
| |
| |
| en_semantic = load_base_file("en_semantic.jsonl", "PENDING_RELABEL", "en", "base_semantic") |
| hi_semantic = load_base_file("hi_semantic.jsonl", "PENDING_RELABEL", "hi", "base_semantic") |
| all_semantic = en_semantic + hi_semantic |
| print(f"Isolated {len(en_semantic)} EN semantic items") |
| print(f"Isolated {len(hi_semantic)} HI semantic items") |
| print(f"Total raw semantic items to relabel: {len(all_semantic)}") |
| |
| semantic_file = os.path.join(OUTPUT_DIR, "semantic_raw.jsonl") |
| with open(semantic_file, "w", encoding="utf-8") as f: |
| for item in all_semantic: |
| f.write(json.dumps(item, ensure_ascii=False) + "\n") |
| print(f"Saved {len(all_semantic)} raw semantic items to {semantic_file}") |
|
|
| if __name__ == "__main__": |
| main() |
|
|
