perf: migrate keywords node to Pydantic structured output for complete recall and zero conversational noise

app.py

@@ -41,7 +41,7 @@ def build_gcmd_indices(gcmd_json):
 VALID_TOPICS, SUB_TREE_LOOKUP = build_gcmd_indices(gcmd_data)
 
 # ==========================================================
-# 2. UPDATED LANGGRAPH WORKFLOW WITH NULL-ENFORCEMENT
+# 2. LANGGRAPH WORKFLOW WITH DUAL STRUCTURED OUTPUTS
 # ==========================================================
 
 def merge_lists(left: list, right: list) -> list:
@@ -54,21 +54,22 @@ class MultiTopicState(TypedDict):
     predicted_keywords: Annotated[List[str], merge_lists]
     invalid_keywords: Annotated[List[str], merge_lists]
 
+# Pydantic schema for Step 1
 class TopicsChoice(BaseModel):
     topics: List[str] = Field(description="List of matching topic areas from the allowed dataset.")
 
+# New Pydantic schema for Step 2 (Guarantees zero mumbling while preserving full recall)
+class KeywordExtraction(BaseModel):
+    keywords: List[str] = Field(description="List of exact matching keyword pathways from the provided text block. Return an empty list if nothing matches.")
+
 def route_multi_topic(state: MultiTopicState):
-    """Step 1: Identify strictly relevant high-level topics (Tightened to prevent over-routing)."""
+    """Step 1: Identify ALL relevant high-level topics (Restored to be healthily inclusive)."""
     llm = ChatOpenAI(model="gpt-4o", temperature=0)
     structured_llm = llm.with_structured_output(TopicsChoice)
     
     prompt = ChatPromptTemplate.from_messages([
-        ("system", (
-            f"You are an expert science cataloger. Identify only the core major topic areas that "
-            f"DIRECTLY and primary apply to this paper. Do NOT select topics that are only tangentially "
-            f"referenced or inferred. Choose ONLY from: {', '.join(VALID_TOPICS)}"
-        )),
-        ("user", "Title: {title}\nAbstract: {abstract}\n\nSelect the highly relevant Topics as a structured list.")
+        ("system", f"You are an expert science cataloger. Identify ALL relevant major topic areas that apply to this paper. Choose ONLY from: {', '.join(VALID_TOPICS)}"),
+        ("user", "Title: {title}\nAbstract: {abstract}\n\nSelect all relevant Topics as a structured list.")
     ])
     
     result = structured_llm.invoke(prompt.format(title=state["title"], abstract=state["abstract"]))
@@ -80,30 +81,24 @@ def route_multi_topic(state: MultiTopicState):
     return {"chosen_topics": valid_selected}
 
 def classify_individual_topic(topic_name: str):
-    """A dynamic factory function with strict enforcement against conversational prose."""
+    """A dynamic factory function using Pydantic tracking to guarantee high recall with zero text pollution."""
     
     def node_runner(state: MultiTopicState):
         llm = ChatOpenAI(model="gpt-4o", temperature=0)
+        # Force strict structured output format
+        structured_llm = llm.with_structured_output(KeywordExtraction)
         target_sub_tree = SUB_TREE_LOOKUP.get(topic_name, "")
         
         prompt = ChatPromptTemplate.from_messages([
-            ("system", (
-                f"You are a specialist in {topic_name} data mapping. Extract exact keyword pathways present in the provided list.\n"
-                f"CRITICAL RULES:\n"
-                f"1. If absolutely no keywords from the valid path list match this paper, reply with exactly the word 'NONE'.\n"
-                f"2. Do NOT write sentences, do NOT explain your reasoning, and do NOT say 'there are no matching entries'. Your output must only be a comma-separated list of keywords, or the single token 'NONE'."
-            )),
-            ("user", "Title: {title}\nAbstract: {abstract}\n\nValid Paths:\n{sub_tree}\n\nReturn exact matching entries as a comma-separated list.")
+            ("system", f"You are a specialist in {topic_name} data mapping. Extract all exact matching keyword pathways present in the provided valid path list. Do not modify or truncate the pathway strings."),
+            ("user", "Title: {title}\nAbstract: {abstract}\n\nValid Paths:\n{sub_tree}\n\nExtract matching pathways as a structured array list.")
         ])
         
-        response = llm.invoke(prompt.format(title=state["title"], abstract=state["abstract"], sub_tree=target_sub_tree))
-        
-        # Split and clean tokens, explicitly ignoring any 'NONE' or empty responses
-        raw_keywords = [
-            k.strip() for k in response.content.split(",") 
-            if k.strip() and k.strip().upper() != "NONE"
-        ]
+        # Enforce JSON list output natively
+        result = structured_llm.invoke(prompt.format(title=state["title"], abstract=state["abstract"], sub_tree=target_sub_tree))
+        raw_keywords = [k.strip() for k in result.keywords if k.strip()]
         
+        # Immediate validation pass inside the node branch
         valid_set = set()
         for line in target_sub_tree.split("\n"):
             if line.strip():