Fix: 10 critical NLP logic bugs in grammar, spelling, and punctuation to prevent false positives

src/nlp/grammar/grammar_rules.py

@@ -159,7 +159,7 @@ class ArabicGrammarGuard:
         disambig_tokens = self.mle.disambiguate(tokens)
 
         nasb_particles = ['أن', 'ان', 'لن', 'كي', 'لكي', 'حتى', 'حتي', 'إذن', 'اذا']
-        jazm_particles = ['لم', 'لما', 'لا']
+        jazm_particles = ['لم', 'لما']
 
         corrected_tokens = []
 
@@ -250,31 +250,36 @@ class ArabicGrammarGuard:
             stem = m.group(2)
             suffix = m.group(3)
             full_word = stem + suffix
-            # Skip words in blocklist (root nouns, not duals)
-            if full_word in self._PREP_BLOCKLIST:
-                return m.group(0)  # return unchanged
-            # Skip ال-prefixed words ending in ان — almost always root nouns
-            if stem.startswith('ال') and suffix == 'ان':
-                return m.group(0)  # return unchanged
-            return f'{prep} {stem}ين'
-
-        text = re.sub(r'\b([وف]?(?:في|من|إلى|على|عن|حتى))\s+([أ-ي]{4,})(ون|ان)\b', _prep_replace, text)
+            
+            # Use camel-tools disambiguation to determine if it's really a dual/plural
+            tokens = simple_word_tokenize(full_word)
+            disambig_tokens = self.mle.disambiguate(tokens)
+            if disambig_tokens and disambig_tokens[0].analyses:
+                num = disambig_tokens[0].analyses[0].analysis.get('num', 's')
+                # Only apply ين suffix if the word is actually Dual or Plural
+                if num in ['d', 'p']:
+                    return f'{prep} {stem}ين'
+            return m.group(0)
+
+        text = re.sub(r'\b([وف]?(?:في|من|إلى|على|عن|حتى))\s+([أ-ي]{3,})(ون|ان)\b', _prep_replace, text)
 
         # (وبالمبرمجون) -> (وبالمبرمجين)
         # FIX-33b: Same blocklist protection as first regex
         def _attached_prep_replace(m):
-            prefix = m.group(1)  # وب، ب، فب، ول، ل، etc.
+            prefix = m.group(1)
             stem = m.group(2)
             suffix = m.group(3)
-            full_word = 'ال' + stem + suffix  # reconstruct with ال for blocklist check
-            if full_word in self._PREP_BLOCKLIST:
-                return m.group(0)
-            # Words ending in ان with 4+ char stems are almost always root nouns
-            if suffix == 'ان':
-                return m.group(0)
-            return f'{prefix}ال{stem}ين'
+            full_word = 'ال' + stem + suffix
+            
+            tokens = simple_word_tokenize(full_word)
+            disambig_tokens = self.mle.disambiguate(tokens)
+            if disambig_tokens and disambig_tokens[0].analyses:
+                num = disambig_tokens[0].analyses[0].analysis.get('num', 's')
+                if num in ['d', 'p']:
+                    return f'{prefix}ال{stem}ين'
+            return m.group(0)
 
-        text = re.sub(r'\b([وف]?[بلكف])ال([أ-ي]{4,})(ون|ان)\b', _attached_prep_replace, text)
+        text = re.sub(r'\b([وف]?[بلكف])ال([أ-ي]{3,})(ون|ان)\b', _attached_prep_replace, text)
 
         # (ولمهندسون) -> (ولمهندسين)
         # FIX-33b: Same protection — reconstruct full word for blocklist
@@ -310,8 +315,8 @@ class ArabicGrammarGuard:
                 if word == 'ذو': return 'ذا'
                 if word == 'فو': return 'فا'
                 if word == 'حمو': return 'حما'
-                if word.endswith('ون') and word not in ('قانون', 'فرعون', 'كانون', 'معجون', 'طاعون', 'مجنون'): return word[:-2] + 'ين'
-                if word.endswith('ان') and word not in ('امتحان', 'إنسان', 'ميدان', 'سلطان', 'شيطان'): return word[:-2] + 'ين'
+                if word.endswith('ون') and num == 'p' and word not in ('قانون', 'فرعون', 'كانون', 'معجون', 'طاعون', 'مجنون'): return word[:-2] + 'ين'
+                if word.endswith('ان') and num == 'd' and word not in ('امتحان', 'إنسان', 'ميدان', 'سلطان', 'شيطان'): return word[:-2] + 'ين'
             elif target_case == 'n': # Marfoo'
                 if word in ('أبا', 'أبي'): return 'أبو'
                 if word in ('أخا', 'أخي'): return 'أخو'
@@ -574,13 +579,13 @@ class ArabicGrammarGuard:
         
         # FIX-PC010: Add targeted safe regex for Nasb/Jazm particles + verb
         # Only match clear present tense verbs starting with ي/ت/ن/أ and ending in ون
-        text = re.sub(r'\b(أن|ان|لن|كي|حتى|لم|لما)\s+([يتا][\u0600-\u06FF]{2,})ون\b',
+        text = re.sub(r'\b(أن|ان|لن|كي|حتى|لم|لما)\s+([يتاأن][\u0600-\u06FF]{2,})ون\b',
                       r'\1 \2وا', text)
 
         return text
 
     def fix_conditional_sentences(self, text):
-        conditional_particles = {'إن', 'ان', 'من', 'ما', 'متى', 'متي', 'مهما', 'أينما', 'حيثما', 'أيان', 'ايان', 'كيفما', 'أنى', 'اني'}
+        conditional_particles = {'إن', 'ان', 'متى', 'متي', 'مهما', 'أينما', 'حيثما', 'أيان', 'ايان', 'كيفما', 'أنى', 'اني'}
         tokens = simple_word_tokenize(text)
         disambig_tokens = self.mle.disambiguate(tokens)
         corrected_tokens = list(tokens)
@@ -700,9 +705,11 @@ class ArabicGrammarGuard:
                 if base_adj:
                     if w1.endswith('ون') or w1.endswith('ين') or w1_gen == 'm':
                         is_nom = w1.endswith('ون')
-                        corrected_tokens[i+1] = base_adj + ('ون' if is_nom else 'ين')
+                        if not w2.endswith('ان') and not w2.endswith('ين') and not w2.endswith('ات'):
+                            corrected_tokens[i+1] = base_adj + ('ون' if is_nom else 'ين')
                     elif w1.endswith('ات') or w1_gen == 'f':
-                        corrected_tokens[i+1] = base_adj + 'ات'
+                        if not w2.endswith('ان') and not w2.endswith('ين') and not w2.endswith('ات'):
+                            corrected_tokens[i+1] = base_adj + 'ات'
 
         return " ".join(corrected_tokens)
 

src/nlp/punctuation/punctuation_rules.py

@@ -91,7 +91,7 @@ def arabic_postprocessing(text: str) -> str:
         if prev_word.startswith(('ال', 'لل', 'بال', 'فال', 'وال', 'كال')):
             return match.group(0)  # Preserve the colon! Do not delete it.
             
-        return match.group(0)
+        return context + ' '
         
     text = re.sub(r'([^:]+)(:)', _colon_guard, text)
     

src/nlp/spelling/araspell_rules.py

@@ -129,14 +129,9 @@ class AraSpellPostProcessor:
     @staticmethod
     def remove_duplicate_words(text: str) -> str:
         """Remove consecutive duplicate words. e.g. كتاب كتاب → كتاب"""
-        words = text.split()
-        if len(words) < 2:
-            return text
-        result = [words[0]]
-        for i in range(1, len(words)):
-            if words[i] != words[i-1]:
-                result.append(words[i])
-        return ' '.join(result)
+        # Bug 2.11: Destroys rhetorical repetition (التوكيد اللفظي) like "صفا صفا".
+        # Disabled as it destroys valid Arabic phrases.
+        return text
     
     @staticmethod
     def normalize_spaces(text: str) -> str:
@@ -337,11 +332,11 @@ class AraSpellPostProcessor:
             if any(word.endswith(e) for e in PROTECTED_ENDINGS):
                 result.append(word)
                 continue
-            if word in PROTECTED_HA_WORDS:
+            if word in PROTECTED_HA_WORDS or word in ['هذه', 'هاته']:
                 result.append(word)
                 continue
             if len(word) >= 3 and word.endswith('ه'):
-                if word[-2] in AraSpellPostProcessor.ARABIC_CONSONANTS:
+                if word[-2] in AraSpellPostProcessor.ARABIC_CONSONANTS or word[-2] in 'اويءؤئ':
                     candidate_with_ta = word[:-1] + 'ة'
                     # Default: prefer ة (correct Arabic orthography for feminine nouns)
                     if vocab_manager:
@@ -389,11 +384,8 @@ class AraSpellPostProcessor:
             if i + 1 < len(words):
                 next_word = words[i + 1]
                 # Bug 2.11: Destroys Badal structures (الأستاذ أستاذ -> الأستاذ)
-                if word == next_word: # Only remove exact duplicates, not normalized duplicates
-                    keep = next_word if next_word.startswith('ال') and not word.startswith('ال') else word
-                    result.append(keep)
-                    i += 2
-                    continue
+                # and Rhetorical Repetition (التوكيد اللفظي)
+                # Removed the aggressive duplicate word deletion.
             result.append(word)
             i += 1
         return ' '.join(result)
@@ -1177,7 +1169,15 @@ class ArabicSpellChecker:
             logger.info("[MLM/CONTEXTUAL] Disabled by configuration (use_contextual=False)")
 
     def _fix_repeated_end_chars(self, text: str) -> str:
-        text = re.sub(r'([ا-ي])\1+\b', r'\1', text)
+        # Exclude 'ي' if it is preceded by a Kasra or another Yaa (e.g., يحيي)
+        def _replace_repeated(m):
+            w = m.group(0)
+            char = m.group(2)
+            if w.endswith('يي'):
+                if self.vocab_manager and self.vocab_manager.is_iv(w):
+                    return w
+            return m.group(1) + char
+        text = re.sub(r'\b([^\s]+?)([\u0621-\u064A])\2+\b', _replace_repeated, text)
         return text
     
     def _fix_merged_with_errors(self, text: str) -> str: