feat: NLP-2 Grammar Integration — dependency-aware pipeline - Add grammar_rules.py (ArabicGrammarGuard from Grammer_Rules.py with camel-tools) - Add grammar_service.py (lazy-loaded Gradio Client + rules pipeline) - Wire /api/grammar endpoint to real grammar service - Wire /api/analyze Step 2 to grammar (receives AraSpell output) - Update /api/health to report grammar status - Add gradio_client to requirements.txt - Add camel_data download to Dockerfile - Frontend unchanged (already supports grammar-error yellow highlighting)

Dockerfile

@@ -45,6 +45,9 @@ AutoModelForMaskedLM.from_pretrained('aubmindlab/bert-base-arabertv02'); \
 print('Spelling model + MLM cached!'); \
 "
 
+# 3. Grammar — camel-tools MLE disambiguator data
+RUN camel_data -i light
+
 # Copy application code
 COPY src/ ./src/
 COPY .env* ./

requirements.txt

@@ -13,3 +13,4 @@ jellyfish
 python-Levenshtein
 gunicorn
 python-dotenv
+gradio_client

src/app.py

@@ -155,7 +155,7 @@ def health_check():
                 'summarization': summarization_model is not None,
                 'spelling': _spelling_available(),
                 'autocomplete': False,
-                'grammar': False,
+                'grammar': _grammar_available(),
                 'punctuation': False
             },
             'note': 'Free tier: summarization local, other models return input unchanged',
@@ -232,6 +232,15 @@ def _spelling_available():
         return False
 
 
+def _grammar_available():
+    """Check if grammar model is loaded (without triggering lazy load)."""
+    try:
+        from nlp.grammar.grammar_service import is_loaded
+        return is_loaded()
+    except Exception:
+        return False
+
+
 @app.route('/api/spelling', methods=['POST'])
 def spelling_correction():
     """
@@ -438,17 +447,18 @@ def grammar_correction():
     """
     Correct grammar in Arabic text.
     
-    Expected JSON payload:
+    Request JSON:
     {
-        "text": "Arabic text to correct"
+        "text": "Arabic text with grammar errors"
     }
-    """
-    if not USE_HF_API and grammar_model is None:
-        return jsonify({
-            'error': 'Grammar model not loaded. Please check server logs.',
-            'status': 'error'
-        }), 503
     
+    Response JSON:
+    {
+        "original_text": "...",
+        "corrected_text": "...",
+        "status": "success"
+    }
+    """
     try:
         if not request.is_json:
             return jsonify({'error': 'Request must be JSON', 'status': 'error'}), 400
@@ -459,20 +469,30 @@ def grammar_correction():
         if not text:
             return jsonify({'error': 'Text is required', 'status': 'error'}), 400
         
-        logger.info(f"Correcting grammar for text of length: {len(text)}")
-        if USE_HF_API:
-            # Grammar uses spelling model as proxy (no dedicated grammar model yet)
-            corrected = hf_correct_spelling(text)
-        else:
-            corrected = grammar_model.correct(text)
+        if len(text) > MAX_TEXT_LENGTH:
+            return jsonify({
+                'error': f'Text too long. Maximum {MAX_TEXT_LENGTH} characters.',
+                'status': 'error'
+            }), 400
+        
+        logger.info(f"Grammar correction request: text_length={len(text)}")
+        
+        from nlp.grammar.grammar_service import get_grammar_model
+        checker = get_grammar_model()
+        corrected = checker.correct(text)
         
         return jsonify({
-            'corrected': corrected,
-            'status': 'success',
-            'original_length': len(text),
-            'corrected_length': len(corrected)
-        })
+            'original_text': text,
+            'corrected_text': corrected,
+            'status': 'success'
+        }), 200
     
+    except RuntimeError as e:
+        logger.error(f"Grammar model error: {e}")
+        return jsonify({
+            'error': f'Grammar model unavailable: {str(e)[:200]}',
+            'status': 'error'
+        }), 503
     except Exception as e:
         logger.error(f"Error during grammar correction: {str(e)}")
         logger.error(traceback.format_exc())
@@ -849,32 +869,29 @@ def analyze_text():
             except Exception as e:
                 logger.error(f"[ANALYZE] Spelling failed: {e}")
 
-        # 2. Grammar (runs on spelling-corrected text)
-        has_grammar = USE_HF_API or grammar_model
-        if has_grammar:
-            try:
-                t0 = time.time()
-                logger.info(f"[ANALYZE] Step 2: Grammar correction starting...")
-                if USE_HF_API:
-                    corrected_grammar = hf_correct_spelling(current_text)
-                else:
-                    corrected_grammar = grammar_model.correct(current_text)
-                logger.info(f"[ANALYZE] Step 2: Grammar done in {time.time()-t0:.2f}s")
-                if corrected_grammar != current_text:
-                    diffs = get_word_diffs(current_text, corrected_grammar)
-                    for d in diffs:
-                        orig_start, orig_end = map_range_to_original(d['start'], d['end'])
-                        suggestions.append({
-                            'start': orig_start,
-                            'end': orig_end,
-                            'original': text[orig_start:orig_end],
-                            'correction': d['correction'],
-                            'type': 'grammar'
-                        })
-                    mappers.append(OffsetMapper(current_text, corrected_grammar))
-                    current_text = corrected_grammar
-            except Exception as e:
-                logger.error(f"[ANALYZE] Grammar failed: {e}")
+        # 2. Grammar (runs on spelling-corrected text — word-level dependency)
+        try:
+            t0 = time.time()
+            logger.info(f"[ANALYZE] Step 2: Grammar correction starting...")
+            from nlp.grammar.grammar_service import get_grammar_model
+            grammar_checker = get_grammar_model()
+            corrected_grammar = grammar_checker.correct(current_text)
+            logger.info(f"[ANALYZE] Step 2: Grammar done in {time.time()-t0:.2f}s")
+            if corrected_grammar != current_text:
+                diffs = get_word_diffs(current_text, corrected_grammar)
+                for d in diffs:
+                    orig_start, orig_end = map_range_to_original(d['start'], d['end'])
+                    suggestions.append({
+                        'start': orig_start,
+                        'end': orig_end,
+                        'original': text[orig_start:orig_end],
+                        'correction': d['correction'],
+                        'type': 'grammar'
+                    })
+                mappers.append(OffsetMapper(current_text, corrected_grammar))
+                current_text = corrected_grammar
+        except Exception as e:
+            logger.error(f"[ANALYZE] Grammar failed: {e}")
 
         # 3. Punctuation (runs on grammar-corrected text)
         has_punctuation = USE_HF_API or punctuation_model

src/nlp/grammar/__init__.py

@@ -0,0 +1 @@
+# Grammar NLP module

src/nlp/grammar/grammar_rules.py

@@ -0,0 +1,187 @@
+# ArabicGrammarGuard — Rule-based Arabic grammar post-processing
+# Extracted from Grammer_Rules.py — uses camel-tools for morphological analysis.
+# All classes are imported by grammar_service.py.
+
+import re
+import logging
+from camel_tools.tokenizers.word import simple_word_tokenize
+from camel_tools.disambig.mle import MLEDisambiguator
+
+logger = logging.getLogger(__name__)
+
+
+class ArabicGrammarGuard:
+    def __init__(self):
+        self.mle = MLEDisambiguator.pretrained()
+
+        self.number_words = ["واحد", "اثنان", "اثنين", "ثلاث", "أربع", "خمس", "ست", "سبع", "ثمان", "تسع", "عشر",
+                             "عشرون", "عشرين", "ثلاثون", "ثلاثين", "أربعون", "أربعين", "خمسون", "خمسين",
+                             "ستون", "ستين", "سبعون", "سبعين", "ثمانون", "ثمانين", "تسعون", "تسعين", "مائة", "ألف"]
+
+        self.asmaa_khamsa_roots = ['اب', 'اخ', 'حم', 'فو', 'ذو']
+
+    def preserve_numbers(self, original_text, generated_text):
+        orig_digits = re.findall(r'\d+', original_text)
+        gen_digits = re.findall(r'\d+', generated_text)
+        if orig_digits and gen_digits and orig_digits != gen_digits:
+            return original_text
+
+        orig_words = [w for w in original_text.split() if any(num in w for num in self.number_words)]
+        gen_words = [w for w in generated_text.split() if any(num in w for num in self.number_words)]
+        if len(orig_words) > 0 and len(gen_words) > 0:
+            if not any(orig[:3] in gen for orig in orig_words for gen in gen_words):
+                 return original_text
+        return generated_text
+
+    def fix_number_and_gender_agreement(self, text):
+        tokens = simple_word_tokenize(text)
+        disambig_tokens = self.mle.disambiguate(tokens)
+        corrected_tokens = list(tokens)
+
+        for i in range(len(disambig_tokens) - 1):
+            w1_info = disambig_tokens[i].analyses[0] if disambig_tokens[i].analyses else None
+            w2_info = disambig_tokens[i+1].analyses[0] if disambig_tokens[i+1].analyses else None
+            if not w1_info or not w2_info: continue
+
+            w1_pos = w1_info.analysis.get('pos', 'unknown')
+            w2_pos = w2_info.analysis.get('pos', 'unknown')
+            w1_word = corrected_tokens[i]
+            w2_word = corrected_tokens[i+1]
+
+            if w1_pos == 'verb' and w2_pos == 'noun':
+                if (w1_word.endswith('ون') or w1_word.endswith('وا')) and (w2_word.endswith('ون') or w2_word.endswith('ين')):
+                    if w1_word.endswith('ون'): corrected_tokens[i] = w1_word[:-2]
+                    elif w1_word.endswith('وا'): corrected_tokens[i] = w1_word[:-2]
+
+            elif w1_pos == 'noun' and w2_pos == 'verb':
+                if w1_word.endswith('ون') and not (w2_word.endswith('ون') or w2_word.endswith('وا') or w2_word.endswith('ين')):
+                    if w2_info.analysis.get('num') == 's':
+                        corrected_tokens[i+1] = w2_word + 'ون'
+
+            # Match adjectives (adj) only; skip words starting with ب or ending with alef tanween
+            elif w1_pos == 'noun' and w2_pos == 'adj':
+                if w1_word.endswith('ون') and not w2_word.endswith('ون'):
+                    if w2_info.analysis.get('num') == 's' and w2_info.analysis.get('gen') == 'm':
+                        if len(w2_word) > 2 and not w2_word.endswith('ا') and not w2_word.startswith('ب'):
+                            corrected_tokens[i+1] = w2_word + 'ون'
+
+        return " ".join(corrected_tokens)
+
+    def smart_asmaa_khamsa_fix(self, text):
+        tokens = simple_word_tokenize(text)
+        disambig_tokens = self.mle.disambiguate(tokens)
+        corrected_tokens = []
+        verb_seen = False
+
+        for i, token_info in enumerate(disambig_tokens):
+            word = tokens[i]
+
+            pos_tag = token_info.analyses[0].analysis.get('pos', 'unknown') if token_info.analyses else 'unknown'
+
+            if pos_tag == 'verb':
+                verb_seen = True
+                corrected_tokens.append(word)
+                continue
+
+            is_asmaa = any(word.startswith(root) or word.startswith('أ' + root[1:]) for root in self.asmaa_khamsa_roots if len(root)>1)
+
+            if is_asmaa and len(word) >= 3:
+                if verb_seen:
+                    word = word.replace('ا', 'و').replace('ي', 'و')
+                    verb_seen = False
+
+            corrected_tokens.append(word)
+
+        return " ".join(corrected_tokens)
+
+    def fix_verbs_nasb_and_jazm(self, text):
+        tokens = simple_word_tokenize(text)
+        disambig_tokens = self.mle.disambiguate(tokens)
+
+        nasb_particles = ['أن', 'لن', 'كي', 'لكي', 'حتى', 'إذن']
+        jazm_particles = ['لم', 'لما', 'لا']
+
+        corrected_tokens = []
+
+        for i, token_info in enumerate(disambig_tokens):
+            word = tokens[i]
+
+            pos_tag = token_info.analyses[0].analysis.get('pos', 'unknown') if token_info.analyses else 'unknown'
+
+            is_nasb_context = False
+            is_jazm_context = False
+
+            if i > 0:
+                prev_word = tokens[i-1]
+                if prev_word in nasb_particles or word.startswith('ل'):
+                    is_nasb_context = True
+                if prev_word in jazm_particles or word.startswith('ل') or word.startswith('ول'):
+                    is_jazm_context = True
+
+            if pos_tag == 'verb' and (is_nasb_context or is_jazm_context):
+                if word.endswith('ون'):
+                    word = word[:-2] + 'وا'
+                elif word.endswith('ان'):
+                    word = word[:-2] + 'ا'
+                elif word.endswith('ين'):
+                    word = word[:-2] + 'ي'
+                elif is_jazm_context:
+                    if word.endswith('و') and len(word) > 3:
+                        word = word[:-1] + 'ُ'
+                    elif (word.endswith('i') or word.endswith('ي')) and len(word) > 3:
+                        if word.endswith('ي'): word = word[:-1] + 'ِ'
+                    elif (word.endswith('ى') or word.endswith('ا')) and len(word) > 3:
+                        word = word[:-1] + 'َ'
+
+            corrected_tokens.append(word)
+        return " ".join(corrected_tokens)
+
+    def fix_gender_agreement(self, text):
+        text = re.sub(r'\bهذان\s+(ال[أ-ي]+تان)\b', r'هاتان \1', text)
+        text = re.sub(r'\bهاتان\s+(ال[أ-ي]+[^ت]ان)\b', r'هذان \1', text)
+        text = re.sub(r'\bهذهن\b', 'هاتان', text)
+
+        text = re.sub(r'\bأحد عشر\s+([أ-ي]+ة)\b', r'إحدى عشرة \1', text)
+        text = re.sub(r'\bأحد عشرة\s+([أ-ي]+ة)\b', r'إحدى عشرة \1', text)
+
+        text = re.sub(r'\bإحدى عشرة\s+([أ-ي]+ا|رجل[اأ]|طالب[اأ]|مهندس[اأ])\b', r'أحد عشر \1', text)
+        text = re.sub(r'\bإحدى عشر\s+([أ-ي]+ا|رجل[اأ]|طالب[اأ]|مهندس[اأ])\b', r'أحد عشر \1', text)
+        return text
+
+    def fix_prepositions_advanced(self, text):
+        # Allow conjunctions (و، ف) before prepositions
+        # (في المهندسون) -> (في المهندسين)
+        text = re.sub(r'\b([وف]?(?:في|من|إلى|على|عن|حتى))\s+([أ-ي]{2,})(ون|ان)\b', r'\1 \2ين', text)
+
+        # (وبالمبرمجون) -> (وبالمبرمجين)
+        text = re.sub(r'\b([وف]?[بلكف])ال([أ-ي]{2,})(ون|ان)\b', r'\1ال\2ين', text)
+
+        # (ولمهندسون) -> (ولمهندسين)
+        text = re.sub(r'\b([وف]?ل)([أ-ي]{2,})(ون|ان)\b', r'\1\2ين', text)
+        return text
+
+    def regex_rules_fallback(self, text):
+        # إن وأخواتها
+        text = re.sub(r'\b(إن|أن|كأن|لكن|لعل|ليت)\s+(أبوك|أخوك|ذو|فوك)\b',
+                      lambda m: f"{m.group(1)} {m.group(2).replace('و', 'ا')}", text)
+
+        # حروف الجر المنفصلة بمسافة (في أخوك -> في أخيك)
+        text = re.sub(r'\b([وف]?(?:في|من|إلى|على|عن))\s+(أبوك|أباك|أخوك|أخاك|ذو|ذا)\b',
+                      lambda m: f"{m.group(1)} {m.group(2).replace('و', 'ي').replace('ا', 'ي')}", text)
+
+        # حروف الجر المتصلة بدون مسافة (بأخوك، لأبوك -> بأخيك، لأبيك)
+        text = re.sub(r'\b([وف]?[بل])(أبوك|أباك|أخوك|أخاك|ذو|ذا)\b',
+                      lambda m: f"{m.group(1)}{m.group(2).replace('و', 'ي').replace('ا', 'ي')}", text)
+        return text
+
+    def process(self, original_text, generated_text):
+        """Apply all grammar rules to model output."""
+        text = self.preserve_numbers(original_text, generated_text)
+        text = self.fix_number_and_gender_agreement(text)
+        text = self.smart_asmaa_khamsa_fix(text)
+        text = self.fix_verbs_nasb_and_jazm(text)
+        text = self.fix_gender_agreement(text)
+        text = self.fix_prepositions_advanced(text)
+        text = self.regex_rules_fallback(text)
+        text = re.sub(r'\s+', ' ', text).strip()
+        return text

src/nlp/grammar/grammar_service.py

@@ -0,0 +1,117 @@
+"""
+Grammar Service — Lazy-loaded Arabic grammar correction.
+
+Uses:
+  1. Gradio Client → mohammedahmedezz2004/bayan_arabic_grammarly_correction (seq2seq model)
+  2. ArabicGrammarGuard (camel-tools rule-based post-processing)
+
+Model + rules loaded on first request and kept in memory.
+"""
+
+import logging
+import time
+
+logger = logging.getLogger(__name__)
+
+# ── Lazy-loaded singletons ──
+_grammar_checker = None
+_load_error = None
+
+GRADIO_SPACE = "mohammedahmedezz2004/bayan_arabic_grammarly_correction"
+
+
+class GrammarChecker:
+    """
+    Grammar correction pipeline:
+      1. Gradio model inference (seq2seq grammar correction)
+      2. Rule-based post-processing (camel-tools ArabicGrammarGuard)
+    """
+
+    def __init__(self, client, rules):
+        self.client = client
+        self.rules = rules
+
+    def correct(self, text: str) -> str:
+        """
+        Run grammar correction on text.
+        Returns corrected text, or original text if correction fails.
+        """
+        if not text or not text.strip():
+            return text
+
+        try:
+            # 1. Model inference via Gradio
+            model_output = self.client.predict(
+                text=text,
+                api_name="/correct_grammar"
+            )
+            logger.info(f"Grammar model output: '{model_output[:80]}...' (input: '{text[:80]}...')")
+
+            if not model_output or not model_output.strip():
+                logger.warning("Grammar model returned empty output, returning original")
+                return text
+
+            # 2. Rule-based post-processing
+            corrected = self.rules.process(text, model_output)
+            logger.info(f"Grammar rules output: '{corrected[:80]}...'")
+
+            return corrected
+
+        except Exception as e:
+            logger.error(f"Grammar correction failed: {e}")
+            # Graceful degradation: return original text
+            return text
+
+
+def get_grammar_model():
+    """
+    Lazy-load the grammar model on first call.
+    Returns the GrammarChecker instance, or raises RuntimeError if loading fails.
+    """
+    global _grammar_checker, _load_error
+
+    if _grammar_checker is not None:
+        return _grammar_checker
+
+    if _load_error is not None:
+        raise RuntimeError(f"Grammar model previously failed to load: {_load_error}")
+
+    try:
+        t0 = time.time()
+        logger.info("Loading Grammar model (lazy init)...")
+
+        # 1. Initialize Gradio Client
+        logger.info(f"Connecting to Gradio Space: {GRADIO_SPACE}")
+        from gradio_client import Client
+        client = Client(GRADIO_SPACE)
+        logger.info("Gradio Client connected")
+
+        # 2. Initialize rule-based post-processor (camel-tools)
+        logger.info("Loading ArabicGrammarGuard (camel-tools MLE disambiguator)...")
+        from nlp.grammar.grammar_rules import ArabicGrammarGuard
+        rules = ArabicGrammarGuard()
+        logger.info("ArabicGrammarGuard loaded")
+
+        # 3. Create GrammarChecker instance
+        _grammar_checker = GrammarChecker(client, rules)
+
+        elapsed = time.time() - t0
+        logger.info(f"Grammar model ready in {elapsed:.1f}s")
+        return _grammar_checker
+
+    except Exception as e:
+        import traceback
+        _load_error = str(e)
+        logger.error(f"Failed to load grammar model: {e}")
+        logger.error(traceback.format_exc())
+        raise RuntimeError(f"Grammar model load failed: {e}")
+
+
+def is_loaded() -> bool:
+    """Check if the grammar model is loaded."""
+    return _grammar_checker is not None
+
+
+def get_load_error() -> str:
+    """Return the last load error, or empty string."""
+    return _load_error or ""