Pushing app.py + requirements.txt + custom dataset

.gitignore

@@ -0,0 +1,3 @@
+venv
+keys.txt
+.gradio\flagged

app.py

@@ -0,0 +1,126 @@
+import os
+import torch
+import random
+import numpy as np
+import pandas as pd
+import gradio as gr
+from pathlib import Path
+
+from transformers import pipeline, set_seed
+from sentence_transformers import SentenceTransformer
+from datasets import load_dataset
+
+# 1. Initialize LLM
+print("Loading BioMistral LLM...")
+llm = pipeline('text-generation', model='BioMistral/BioMistral-7B')
+
+# 2. Load Datasets
+print("Loading datasets...")
+dataset = load_dataset("cbasu/Med-EASi", split="train")
+dataset2 = load_dataset("cbasu/Med-EASi", split="validation")
+dataset3 = load_dataset("cbasu/Med-EASi", split="test")
+dataset4 = load_dataset("csv", data_files="custom_terms.csv", split="train")
+
+# 3. Format dataset rows into strings
+texts = []
+for row in dataset:
+    content = f"Medical Context: {row['Expert']}\nSimplified Explanation: {row['Simple']}\nTerms: {row['expert_terms']}, {row['layman_terms']}"
+    texts.append(content)
+for row in dataset2:
+    content = f"Medical Context: {row['Expert']}\nSimplified Explanation: {row['Simple']}\nTerms: {row['expert_terms']}, {row['layman_terms']}"
+    texts.append(content)
+for row in dataset3:
+    content = f"Medical Context: {row['Expert']}\nSimplified Explanation: {row['Simple']}\nTerms: {row['expert_terms']}, {row['layman_terms']}"
+    texts.append(content)
+for row in dataset4:
+    content = f"Medical Context: {row['Expert']}\nSimplified Explanation: {row['Simple']}\nTerms: {row['expert_terms']}, {row['layman_terms']}"
+    texts.append(content)
+
+print(f"Total processed documents: {len(texts)}")
+
+# 4. Generate Embeddings & Index
+print("Indexing dataset embeddings...")
+embedding_model = SentenceTransformer('all-MiniLM-L6-v2')
+embeddings = embedding_model.encode(texts)
+
+df = pd.DataFrame({
+    "Document": texts,
+    "Embedding": list(embeddings)
+})
+
+# 5. Retrieval Helper Function
+def retrieve_with_pandas(query, top_k=3):
+    query_embedding = embedding_model.encode([query])[0]
+    df['Similarity'] = df['Embedding'].apply(
+        lambda x: np.dot(query_embedding, x) / (np.linalg.norm(query_embedding) * np.linalg.norm(x))
+    )
+    results = df.sort_values(by="Similarity", ascending=False).head(top_k)
+    return results["Document"].tolist()
+
+# 6. Prompt Builder Helper Function
+def build_prompt(context_docs: list, user_query: str) -> str:
+    context = "\n\n".join(context_docs)
+    prompt = f"""<s>[INST] You are a helpful medical assistant.
+Use the following retrieved medical context to answer the user's question clearly and concisely.
+Explain it in very simple terms as if the user has the vocabulary of an elementary school student.
+If the context does not contain enough information to answer, state that you don't know based on the provided text.
+Answer in 2 to 3 sentences maximum.
+
+Context:
+{context}
+
+Question:
+{user_query} [/INST]"""
+    return prompt
+
+# 7. Answer Generation Helper Function
+def generate_answer(prompt: str, max_new_tokens: int = 150) -> str:
+    set_seed(42)
+
+    if llm.model.config.pad_token_id is None:
+        llm.model.config.pad_token_id = llm.model.config.eos_token_id
+
+    output = llm(
+        prompt,
+        max_new_tokens=max_new_tokens,
+        num_return_sequences=1,
+        do_sample=False,
+    )[0]["generated_text"]
+
+    # Strip out the prompt text to leave only the model's new response
+    answer = output[len(prompt):].strip()
+    return answer
+
+# 8. RAG Handler Function
+def respond_to_query(user_query: str) -> str:
+    if not user_query.strip():
+        return "Please enter a valid medical question or term."
+    
+    # Step A: Retrieve context documents based on user input
+    context_docs = retrieve_with_pandas(user_query, top_k=3)
+    
+    # Step B: Build prompt with retrieved context
+    prompt = build_prompt(context_docs, user_query)
+    
+    # Step C: Generate model answer
+    answer = generate_answer(prompt, max_new_tokens=150)
+    
+    return answer
+
+# 9. Gradio Web Interface
+demo = gr.Interface(
+    fn=respond_to_query,
+    inputs=gr.Textbox(lines=2, placeholder="e.g., What is necrosis? or What is cholera?"),
+    outputs=gr.Textbox(label="BioMistral RAG Response", lines=4),
+    title="🩺 Medical Terminology RAG Assistant",
+    description="Ask a medical question to retrieve context from Med-EASi + custom terms and generate simplified explanations.",
+    examples=[
+        ["What causes necrosis?"],
+        ["What is cholera?"],
+        ["What is asbestosis?"],
+        ["How can syphilis be treated?"]
+    ]
+)
+
+if __name__ == "__main__":
+    demo.launch()

custom_terms.csv

@@ -0,0 +1,11 @@
+Expert,Simple,expert_terms,layman_terms
+Necrosis is a form of cell injury which results in the premature death of cells in living tissue by destruction of the cell through the action of its own enzymes,"Unnatural cell death caused by severe damage, where cells are broken down by themselves.",necrosis,cell death
+"Anaphylaxis is a severe, systemic, and potentially life-threatening allergic reaction that occurs rapidly following exposure to an allergen.","A sudden, severe, and dangerous allergic reaction that affects the whole body.",anaphylaxis,severe allergic reaction
+"Ischemia is a restriction in blood supply to tissues, causing a shortage of oxygen needed for cellular metabolism.",A lack of adequate blood flow and oxygen to a part of the body.,ischemia,restricted blood flow
+Thoracentesis is a procedure in which a needle is inserted into the pleural space between the lungs and chest wall to remove excess fluid.,A medical procedure where a doctor uses a needle to drain fluid from around the lungs.,thoracentesis,lung fluid removal
+Thrombolysis is the breakdown or dissolution of blood clots formed in blood vessels using pharmacological agents.,The use of special medication to quickly dissolve dangerous blood clots.,thrombolysis,clot dissolving
+"Atelectasis is a complete or partial collapse of the entire lung or lobe of the lung, occurring when the alveoli become deflated or filled with alveolar fluid.",A condition where part or all of a lung collapses because the air sacs deflate.,atelectasis,collapsed lung
+"Kinetosis is a neurological condition caused by a sensory conflict between the vestibular, visual, and proprioceptive systems during real or perceived motion.","A feeling of nausea, dizziness, or cold sweats when your brain gets conflicting signals from your eyes and inner ears about movement.",kinetosis,motion sickness
+"Dyspnea is the clinical term for shortness of breath, often described as an intense tightening in the chest, air hunger, or difficulty breathing.",Difficulty breathing or feeling short of breath.,dyspnea,shortness of breath
+Idiopathic describes any disease or condition that arises spontaneously and for which the cause is currently unknown or obscure.,A medical condition or symptom that happens without any known cause.,idiopathic,unknown cause
+Metastasis is the pathogenic process by which cancer cells spread from the primary site of origin to distant tissues or organs.,The process where cancer cells spread from where they started to other parts of the body.,metastasis,cancer spread

requirements.txt

@@ -0,0 +1,7 @@
+transformers
+sentence-transformers
+datasets
+pandas
+numpy
+torch
+gradio