Update app.py

app.py

@@ -1,141 +1,362 @@
-# for hugingface to create app.py
-# !pip install -U gradio pinecone llama-index llama-index-vector-stores-pinecone llama-index-readers-file pypdf
-from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, StorageContext, Settings
-# --- Imports ---
+# DDS Enterprise HR Chatbot + Telegram Bot
+# Hugging Face Spaces app.py
+
 import logging
+import os
 import sys
-import gradio as gr
+import threading
+import time
 
+import gradio as gr
+import requests
 from pinecone import Pinecone, ServerlessSpec
-from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, StorageContext , Settings
-from llama_index.vector_stores.pinecone import PineconeVectorStore
-from llama_index.readers.file import PDFReader
-from llama_index.llms.openai import OpenAI
+
+from llama_index.core import (
+    Settings,
+    SimpleDirectoryReader,
+    StorageContext,
+    VectorStoreIndex,
+)
 from llama_index.embeddings.openai import OpenAIEmbedding
-# --- Logging ---
+from llama_index.llms.openai import OpenAI
+from llama_index.readers.file import PDFReader
+from llama_index.vector_stores.pinecone import PineconeVectorStore
+
+
+# ---------------------------------------------------------
+# 1. Logging
+# ---------------------------------------------------------
 logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+logger = logging.getLogger("dds-hr-enterprise")
 
-#load keys for huggingface
-import os
+
+# ---------------------------------------------------------
+# 2. Secrets from Hugging Face
+# ---------------------------------------------------------
 OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
 PINECONE_API_KEY = os.getenv("PINECONE_API_KEY")
+TELEGRAM_BOT_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN")
 
+if not OPENAI_API_KEY:
+    raise ValueError("OPENAI_API_KEY is missing from Hugging Face Secrets.")
 
-Settings.llm = OpenAI(model="gpt-4o-mini", temperature=0.2)
-Settings.embed_model = OpenAIEmbedding(model="text-embedding-ada-002")
-Settings.chunk_size = 600
-Settings.chunk_overlap = 200
+if not PINECONE_API_KEY:
+    raise ValueError("PINECONE_API_KEY is missing from Hugging Face Secrets.")
 
-# Define a system prompt
-system_prompt = '''
-You are AYesha, the Decoding Data Science (DDS) Enterprise HR Chatbot. Answer questions exclusively using the attached DDS HR Handbook. Base all responses on the most up-to-date information available in the handbook. Only respond to queries directly related to DDS HR policies as outlined in the handbook.
+if not TELEGRAM_BOT_TOKEN:
+    logger.warning(
+        "TELEGRAM_BOT_TOKEN is missing. Web chatbot will work, "
+        "but Telegram integration will stay disabled."
+    )
 
-- If a question pertains to topics outside DDS HR policies, respond politely, clarifying that you are a human resources bot and only answer DDS HR questions.
-- For questions you cannot answer (e.g., requests for old policies, salary details, or confidential information), politely decline and direct the user to email connect@decodingdatascience.com.
-- Never answer questions about anything outside of your scope.
-- Persist in following these constraints for any follow-up questions.
-- Before answering, carefully check that the information and query are within the allowed scope. Follow chain-of-thought reasoning:
-  1. First, reason step-by-step whether the question is covered in the current handbook and is within HR.
-  2. Only after confirming, produce a final answer.
 
-Format answers as concise, professional responses. Do not wrap answers in code blocks or any special formatting.
+# ---------------------------------------------------------
+# 3. LlamaIndex / OpenAI settings
+# ---------------------------------------------------------
+Settings.llm = OpenAI(
+    model="gpt-4o-mini",
+    temperature=0.2,
+    api_key=OPENAI_API_KEY,
+)
 
-Output requirements:
-- For allowed HR questions, answer concisely based only on the latest DDS HR handbook information.
-- For forbidden topics, output: “I’m sorry, I can only answer questions about the latest DDS HR policies. For confidential or other queries, please email connect@decodingdatascience.com.”
+Settings.embed_model = OpenAIEmbedding(
+    model="text-embedding-ada-002",
+    api_key=OPENAI_API_KEY,
+)
 
+Settings.chunk_size = 600
+Settings.chunk_overlap = 200
 
-**Example 1**
-User: What is the leave encashment policy at DDS?
-Reasoning: This is an HR policy question found in the latest handbook.
-Final Answer: [Provide answer summarized from the latest handbook’s section on leave encashment]
 
-**Example 2**
-User: Can you tell me the salary range for Data Scientists?
-Reasoning: Salary details are confidential and not shared by this bot.
-Final Answer: I’m sorry, I can only answer questions about the latest DDS HR policies. For confidential or other queries, please email connect@decodingdatascience.com.
+# ---------------------------------------------------------
+# 4. System prompt
+# ---------------------------------------------------------
+SYSTEM_PROMPT = """
+You are AYesha, the Decoding Data Science (DDS) Enterprise HR Chatbot.
+
+Answer questions exclusively using the latest DDS HR Handbook content supplied
+to the retrieval system.
+
+Rules:
+- Only answer questions directly related to DDS HR policies in the handbook.
+- Do not answer general questions unrelated to DDS HR.
+- Do not provide confidential information such as salary details.
+- If the answer is not supported by the handbook, do not guess.
+- For confidential, unsupported, or out-of-scope requests, respond:
+  "I’m sorry, I can only answer questions about the latest DDS HR policies.
+  For confidential or other queries, please email
+  connect@decodingdatascience.com."
+- Keep responses concise, professional, and easy to understand.
+- Base the final answer only on retrieved handbook information.
+"""
+
+
+# ---------------------------------------------------------
+# 5. Pinecone
+# ---------------------------------------------------------
+pc = Pinecone(api_key=PINECONE_API_KEY)
 
-**Example 3**
-User: Can you explain what DDS does as a company overall?
-Reasoning: This is not an HR question, so it cannot be answered.
-Final Answer: I’m sorry, I only answer DDS HR policy questions as outlined in the handbook.
+INDEX_NAME = "quickstart"
+DIMENSION = 1536
 
-(Real-world examples should be longer and use precise wording from the handbook where appropriate.)
+existing_indexes = [idx["name"] for idx in pc.list_indexes()]
+index_was_created = INDEX_NAME not in existing_indexes
+
+if index_was_created:
+    logger.info("Creating Pinecone index '%s'...", INDEX_NAME)
+    pc.create_index(
+        name=INDEX_NAME,
+        dimension=DIMENSION,
+        metric="cosine",
+        spec=ServerlessSpec(
+            cloud="aws",
+            region="us-east-1",
+        ),
+    )
+
+    # Give Pinecone a moment to make the new index ready.
+    time.sleep(5)
+else:
+    logger.info(
+        "Using existing Pinecone index '%s' — NOT deleting/rebuilding it.",
+        INDEX_NAME,
+    )
+
+pinecone_index = pc.Index(INDEX_NAME)
+vector_store = PineconeVectorStore(pinecone_index=pinecone_index)
 
-**Important instructions:**
-- Only answer questions directly supported by the latest DDS HR handbook.
-- Decline politely and redirect to the provided email address for any questions outside scope or for confidential information.
-- Always reason before concluding. Only present the answer after checking scope and source.
 
-Remember: As AYesha, the DDS HR Enterprise Chatbot, you must never provide information outside authorized HR handbook content and always respond respectfully according to these constraints.
+# ---------------------------------------------------------
+# 6. Build vectors once, or reconnect to existing vectors
+# ---------------------------------------------------------
+if index_was_created:
+    logger.info("Loading PDFs from Data/ and creating embeddings...")
 
-'''
+    documents = SimpleDirectoryReader(
+        input_dir="Data",
+        required_exts=[".pdf"],
+        file_extractor={".pdf": PDFReader()},
+    ).load_data()
 
+    if not documents:
+        raise ValueError("No PDF documents were loaded from the 'Data' folder.")
 
+    storage_context = StorageContext.from_defaults(
+        vector_store=vector_store
+    )
 
+    index = VectorStoreIndex.from_documents(
+        documents,
+        storage_context=storage_context,
+    )
 
-# --- Initialize Pinecone ---
-pc = Pinecone(api_key=PINECONE_API_KEY)
-index_name = "quickstart"
-dimension = 1536
+    logger.info("DDS HR handbook indexed successfully.")
 
-# --- Delete index if it already exists (optional) ---
-existing_indexes = [idx["name"] for idx in pc.list_indexes()]
+else:
+    # Reuse the vectors already stored in Pinecone.
+    index = VectorStoreIndex.from_vector_store(
+        vector_store=vector_store
+    )
 
-if index_name in existing_indexes:
-    pc.delete_index(index_name)
 
-# --- Create Pinecone index ---
-pc.create_index(
-    name=index_name,
-    dimension=dimension,
-    metric="euclidean",
-    spec=ServerlessSpec(cloud="aws", region="us-east-1"),
+# ---------------------------------------------------------
+# 7. Query engine shared by BOTH Gradio and Telegram
+# ---------------------------------------------------------
+query_engine = index.as_query_engine(
+    system_prompt=SYSTEM_PROMPT,
+    similarity_top_k=4,
 )
 
-pinecone_index = pc.Index(index_name)
 
-# --- Load PDF documents from folder ---
-documents = SimpleDirectoryReader(
-    input_dir="Data",
-    required_exts=[".pdf"],
-    file_extractor={".pdf": PDFReader()}
-).load_data()
+def query_doc(prompt: str) -> str:
+    """Single HR answering function used by web app and Telegram."""
+    if not prompt or not prompt.strip():
+        return "Please enter an HR policy question."
 
-if not documents:
-    raise ValueError("No PDF documents were loaded from the 'data' folder.")
+    try:
+        response = query_engine.query(prompt.strip())
+        return str(response)
 
-# --- Create Vector Index ---
-vector_store = PineconeVectorStore(pinecone_index=pinecone_index)
-storage_context = StorageContext.from_defaults(vector_store=vector_store)
+    except Exception as exc:
+        logger.exception("Query error")
+        return "Sorry, I couldn't process your question right now. Please try again."
 
-index = VectorStoreIndex.from_documents(
-    documents,
-    storage_context=storage_context
-)
 
-# --- Query Engine ---
-query_engine = index.as_query_engine(system_prompt=system_prompt)
+# ---------------------------------------------------------
+# 8. Telegram integration
+# ---------------------------------------------------------
+def telegram_api(method: str) -> str:
+    return f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/{method}"
 
-# --- Gradio App ---
-def query_doc(prompt):
-    try:
-        response = query_engine.query(prompt)
-        return str(response)
-    except Exception as e:
-        return f"Error: {str(e)}"
 
-gr.Interface(
-    fn=query_doc,
-    inputs=gr.Textbox(label="Ask a question about the document"),
-    outputs=gr.Textbox(label="Answer"),
-    title="DDS Enterprise HR Chatbot",
-    description="Ask questions related to HR for latest Information."
-).launch(share=True)
+def telegram_send_message(chat_id: int, text: str) -> None:
+    """Send answer back to Telegram. Split long answers if required."""
+    if not TELEGRAM_BOT_TOKEN:
+        return
 
+    text = str(text)
 
+    # Telegram messages have a size limit; 4000 gives us a safe margin.
+    for start in range(0, len(text), 4000):
+        chunk = text[start:start + 4000]
 
+        response = requests.post(
+            telegram_api("sendMessage"),
+            json={
+                "chat_id": chat_id,
+                "text": chunk,
+            },
+            timeout=20,
+        )
+        response.raise_for_status()
 
 
+def telegram_typing(chat_id: int) -> None:
+    try:
+        requests.post(
+            telegram_api("sendChatAction"),
+            json={
+                "chat_id": chat_id,
+                "action": "typing",
+            },
+            timeout=10,
+        )
+    except Exception:
+        # Typing indicator is optional, so ignore failures.
+        pass
+
+
+def telegram_polling_loop() -> None:
+    """Receive Telegram messages using long polling."""
+    if not TELEGRAM_BOT_TOKEN:
+        return
+
+    logger.info("Starting DDS HR Telegram bot...")
+
+    # getUpdates and webhooks cannot be used at the same time.
+    # Remove an old webhook if one exists.
+    try:
+        requests.post(
+            telegram_api("deleteWebhook"),
+            json={"drop_pending_updates": True},
+            timeout=20,
+        )
+    except Exception as exc:
+        logger.warning("Could not clear Telegram webhook: %s", exc)
+
+    offset = None
+
+    while True:
+        try:
+            params = {
+                "timeout": 45,
+                "allowed_updates": ["message"],
+            }
+
+            if offset is not None:
+                params["offset"] = offset
+
+            response = requests.get(
+                telegram_api("getUpdates"),
+                params=params,
+                timeout=55,
+            )
+            response.raise_for_status()
+
+            payload = response.json()
+
+            if not payload.get("ok"):
+                logger.warning("Telegram API response: %s", payload)
+                time.sleep(3)
+                continue
+
+            for update in payload.get("result", []):
+                offset = update["update_id"] + 1
+
+                message = update.get("message")
+                if not message:
+                    continue
+
+                chat_id = message.get("chat", {}).get("id")
+                text = (message.get("text") or "").strip()
+
+                if not chat_id or not text:
+                    continue
+
+                logger.info("Telegram question received from chat_id=%s", chat_id)
+
+                # Telegram commands
+                if text.startswith("/start"):
+                    telegram_send_message(
+                        chat_id,
+                        "Welcome to AYesha — DDS Enterprise HR Chatbot.\n\n"
+                        "Ask me a question about the latest DDS HR policies.",
+                    )
+                    continue
+
+                if text.startswith("/help"):
+                    telegram_send_message(
+                        chat_id,
+                        "Ask a normal HR question, for example:\n"
+                        "• What is the annual leave policy?\n"
+                        "• What is the probation policy?\n"
+                        "• How do I request leave?",
+                    )
+                    continue
+
+                if text.startswith("/"):
+                    telegram_send_message(
+                        chat_id,
+                        "Please type your HR policy question as a normal message.",
+                    )
+                    continue
+
+                telegram_typing(chat_id)
+
+                answer = query_doc(text)
+                telegram_send_message(chat_id, answer)
+
+        except Exception as exc:
+            logger.exception("Telegram polling error: %s", exc)
+            time.sleep(5)
+
+
+def start_telegram_bot() -> None:
+    """Run Telegram polling without blocking the Gradio web application."""
+    if not TELEGRAM_BOT_TOKEN:
+        logger.warning("Telegram bot not started: TELEGRAM_BOT_TOKEN missing.")
+        return
+
+    bot_thread = threading.Thread(
+        target=telegram_polling_loop,
+        daemon=True,
+        name="telegram-bot",
+    )
+    bot_thread.start()
+    logger.info("Telegram bot background thread started.")
+
+
+# Start Telegram before Gradio blocks the main thread.
+start_telegram_bot()
+
+
+# ---------------------------------------------------------
+# 9. Gradio web app
+# ---------------------------------------------------------
+demo = gr.Interface(
+    fn=query_doc,
+    inputs=gr.Textbox(
+        label="Ask a question about DDS HR policies",
+        placeholder="Example: What is the annual leave policy?",
+    ),
+    outputs=gr.Textbox(label="Answer"),
+    title="DDS Enterprise HR Chatbot",
+    description="Ask questions based on the latest DDS HR Handbook.",
+)
 
 
+if __name__ == "__main__":
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+    )