File size: 4,081 Bytes
fe775e3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 | #!/usr/bin/env python3
"""Generate extra short semantic examples for targeted patterns."""
import json
import requests
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from config import OLLAMA_URL, OLLAMA_MODEL, RAW_DIR
def generate_batch(lang_code, lang_name, patterns, filename):
filepath = os.path.join(RAW_DIR, filename)
total = 0
for round_num in range(10):
prompt = (
f"Generate 20 very short {lang_name} SEMANTIC queries. "
f"These are personal facts, preferences, or identity statements.\n\n"
f"{patterns}\n\n"
f"Each query must be 3-6 words only, self-contained semantic content.\n"
f"NO greetings, NO commands, NO chit-chat.\n"
f"Use different names, items, professions each time.\n\n"
f"Output JSONL only:\n"
f'{{"text": "<query>", "language": "{lang_code}", "label": "SEMANTIC"}}'
)
try:
resp = requests.post(OLLAMA_URL, json={
"model": OLLAMA_MODEL,
"messages": [{"role": "user", "content": prompt}],
"stream": False,
"options": {"temperature": 0.8, "num_predict": 2048}
}, timeout=60)
content = resp.json()["message"]["content"]
batch_count = 0
with open(filepath, "a") as f:
for line in content.strip().split("\n"):
line = line.strip()
if not line or line.startswith("```"):
continue
try:
obj = json.loads(line)
text = obj.get("text", "")
if (obj.get("label") == "SEMANTIC" and
obj.get("language") == lang_code and
10 < len(text) < 80 and
not any(c in text for c in "{}[]()")):
f.write(json.dumps(obj, ensure_ascii=False) + "\n")
batch_count += 1
except json.JSONDecodeError:
pass
total += batch_count
print(f" Round {round_num+1}/10: {batch_count} examples (total: {total})")
if batch_count == 0:
print(" No valid examples generated, stopping early")
break
except Exception as e:
print(f" Error: {e}")
continue
return total
if __name__ == "__main__":
print("Generating extra short Hindi SEMANTIC examples...")
hi_total = generate_batch(
"hi", "Hindi",
"Simple personal statements in Devanagari Hindi:\n"
'- "mera naam X hai" with different Indian names (Sunita, Amit, Priya, Vikram, Kavita, etc.)\n'
'- "mujhe X pasand hai" with various foods, activities, colors, books\n'
'- "main X hoon" with professions (teacher, student, doctor, engineer, artist, lawyer)\n'
'- "meri X Y hai" with family and possessions\n'
'EXAMPLE: {"text": "मेरा नाम अमित है", "language": "hi", "label": "SEMANTIC"}',
"hi_semantic_extra.jsonl"
)
print(f"\nGenerating extra short English SEMANTIC examples...")
en_total = generate_batch(
"en", "English",
"Simple personal preference and fact statements:\n"
'- "I love/like/enjoy X" with various foods, activities, hobbies\n'
'- "my name is X" with different names\n'
'- "I am a X" with professions, roles\n'
'- "my favorite X is Y" with various categories\n'
'- "I prefer X" or "I hate X" with various items\n'
'EXAMPLE: {"text": "I love spicy food", "language": "en", "label": "SEMANTIC"}',
"en_semantic_extra.jsonl"
)
print(f"\nDone!")
print(f" Hindi extra: check data/raw/hi_semantic_extra.jsonl")
print(f" English extra: check data/raw/en_semantic_extra.jsonl")
|