feat: dialect-to-MSA conversion — backend API + frontend + Docker model cache

Dockerfile

@@ -59,6 +59,18 @@ EncoderDecoderModel.from_pretrained(repo); \
 print('PuncAra-v1 cached!'); \
 "
 
+# 5. Dialect-to-MSA model (mT5, float16)
+RUN python -c "\
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM; \
+import torch; \
+repo = 'bayan10/dialect-to-msa-model'; \
+print('Downloading dialect tokenizer...'); \
+AutoTokenizer.from_pretrained(repo); \
+print('Downloading dialect model (float16)...'); \
+AutoModelForSeq2SeqLM.from_pretrained(repo, torch_dtype=torch.float16); \
+print('Dialect model cached!'); \
+"
+
 # Copy application code
 COPY src/ ./src/
 COPY quran.py ./

src/app.py

@@ -174,7 +174,8 @@ def health_check():
                 'spelling': _spelling_available(),
                 'autocomplete': _autocomplete_available(),
                 'grammar': _grammar_available(),
-                'punctuation': _punctuation_available()
+                'punctuation': _punctuation_available(),
+                'dialect': _dialect_available()
             },
             'note': 'Free tier: summarization local, other models return input unchanged',
             'supabase': {
@@ -193,7 +194,8 @@ def health_check():
             'spelling': spelling_model is not None,
             'autocomplete': autocomplete_model is not None,
             'grammar': grammar_model is not None,
-            'punctuation': punctuation_model is not None
+            'punctuation': punctuation_model is not None,
+            'dialect': _dialect_available()
         },
         'supabase': {
             'configured': bool(SUPABASE_URL and SUPABASE_ANON_KEY),
@@ -277,6 +279,15 @@ def _autocomplete_available():
         return False
 
 
+def _dialect_available():
+    """Check if dialect model is loaded (without triggering lazy load)."""
+    try:
+        from nlp.dialect.dialect_service import is_loaded
+        return is_loaded()
+    except Exception:
+        return False
+
+
 @app.route('/api/spelling', methods=['POST'])
 def spelling_correction():
     """
@@ -1073,6 +1084,71 @@ def _is_orthographic_variant(word1: str, word2: str) -> bool:
     return diff_count > 0  # At least one orthographic difference
 
 
+@app.route('/api/dialect', methods=['POST'])
+def convert_dialect():
+    """
+    Convert dialect Arabic text to Modern Standard Arabic (MSA).
+
+    Request JSON:
+    {
+        "text": "عايز اشتكي من موظف في فرعكم"
+    }
+
+    Response JSON:
+    {
+        "status": "success",
+        "original_text": "...",
+        "converted_text": "..."
+    }
+    """
+    try:
+        if not request.is_json:
+            return jsonify({'error': 'Request must be JSON', 'status': 'error'}), 400
+
+        data = request.get_json()
+        text = data.get('text', '').strip()
+
+        if not text:
+            return jsonify({'error': 'Text is required', 'status': 'error'}), 400
+
+        if len(text) > MAX_TEXT_LENGTH:
+            return jsonify({
+                'error': f'Text too long. Maximum {MAX_TEXT_LENGTH} characters.',
+                'status': 'error'
+            }), 400
+
+        logger.info(f"[DIALECT] Conversion request: text_length={len(text)}")
+
+        from nlp.dialect.dialect_service import get_dialect_model
+        converter = get_dialect_model()
+        t0 = time.time()
+        result = converter.convert(text)
+        elapsed = int((time.time() - t0) * 1000)
+
+        logger.info(f"[DIALECT] {elapsed}ms | input='{text[:80]}' | output='{result[:80]}'")
+
+        return jsonify({
+            'original_text': text,
+            'converted_text': result,
+            'status': 'success'
+        }), 200
+
+    except RuntimeError as e:
+        logger.error(f"Dialect model error: {e}")
+        return jsonify({
+            'error': f'Dialect model unavailable: {str(e)[:200]}',
+            'status': 'error'
+        }), 503
+    except Exception as e:
+        logger.error(f"Error during dialect conversion: {e}")
+        logger.error(traceback.format_exc())
+        return jsonify({
+            'error': 'An error occurred during dialect conversion.',
+            'status': 'error',
+            'details': str(e) if app.debug else None
+        }), 500
+
+
 @app.route('/api/quran', methods=['POST'])
 def quran_verify():
     """

src/index.html

@@ -865,7 +865,10 @@
           <div id="dialect-result-card" class="is-hidden" style="background: var(--color-surface); border: 1px solid var(--color-border); border-radius: 1rem; padding: 1.5rem;">
            <div class="flex items-center justify-between mb-3">
             <h3 class="text-base font-bold" style="color: var(--color-primary);">✓ النص بالعربية الفصحى</h3>
-            <button onclick="copyDialectResult()" class="btn-ghost text-sm" type="button">نسخ</button>
+            <div class="flex items-center gap-2">
+             <button onclick="copyDialectResult()" class="quran-copy-btn" type="button" title="نسخ">📋</button>
+             <button id="dialect-apply-btn" onclick="applyDialectResult()" class="quran-apply-btn is-hidden" type="button">تطبيق في المحرر ✓</button>
+            </div>
            </div>
            <div id="dialect-result" class="text-right text-lg editor-content" dir="rtl" style="line-height: 2;"></div>
           </div>
@@ -1192,8 +1195,57 @@
       else if (tab === 'summarize') { summarizeTab.classList.add('active'); summarizeArea.classList.remove('is-hidden'); if(formatToolbar)formatToolbar.style.display='none'; }
       else if (tab === 'dialect') { dialectTab.classList.add('active'); dialectArea.classList.remove('is-hidden'); if(formatToolbar)formatToolbar.style.display='none'; }
     }
-    function convertDialect(){var i=document.getElementById('dialect-input').value.trim();if(!i){if(typeof showToast==='function')showToast('الرجاء كتابة نص أولاً');return;}var r=document.getElementById('dialect-result-card');var d=document.getElementById('dialect-result');d.innerHTML='<p class="text-secondary text-center">⏳ جاري التحويل...</p>';r.classList.remove('is-hidden');setTimeout(function(){d.innerHTML='<p class="text-secondary text-center">🚧 هذه الميزة قيد التطوير — ستتوفر قريبًا</p>';},1000);}
-    function copyDialectResult(){var t=document.getElementById('dialect-result');if(t){navigator.clipboard.writeText(t.innerText);if(typeof showToast==='function')showToast('✓ تم النسخ');}}
+    let _dialectResult = '';
+    async function convertDialect() {
+      var input = document.getElementById('dialect-input').value.trim();
+      if (!input) { if (typeof showToast === 'function') showToast('الرجاء كتابة نص أولاً'); return; }
+
+      var resultCard = document.getElementById('dialect-result-card');
+      var resultDiv = document.getElementById('dialect-result');
+      var applyBtn = document.getElementById('dialect-apply-btn');
+      resultDiv.innerHTML = '<p class="text-secondary text-center">⏳ جاري التحويل...</p>';
+      resultCard.classList.remove('is-hidden');
+      if (applyBtn) applyBtn.classList.add('is-hidden');
+
+      try {
+        var resp = await fetch('/api/dialect', {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({ text: input })
+        });
+        var data = await resp.json();
+        if (data.status === 'success' && data.converted_text) {
+          _dialectResult = data.converted_text;
+          var esc = function(t) { return t.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;'); };
+          resultDiv.innerHTML = '<p style="font-size:20px; line-height:2; direction:rtl; text-align:center;">' + esc(data.converted_text) + '</p>';
+          if (applyBtn) applyBtn.classList.remove('is-hidden');
+        } else {
+          _dialectResult = '';
+          resultDiv.innerHTML = '<p class="text-secondary text-center">' + (data.error || 'حدث خطأ أثناء التحويل') + '</p>';
+        }
+      } catch (err) {
+        _dialectResult = '';
+        resultDiv.innerHTML = '<p class="text-secondary text-center">حدث خطأ — تأكد من الاتصال</p>';
+      }
+    }
+
+    function copyDialectResult() {
+      if (!_dialectResult) { var t = document.getElementById('dialect-result'); if (t) navigator.clipboard.writeText(t.innerText); }
+      else navigator.clipboard.writeText(_dialectResult);
+      if (typeof showToast === 'function') showToast('✓ تم النسخ');
+    }
+
+    function applyDialectResult() {
+      if (!_dialectResult) return;
+      var editor = document.getElementById('editor-container');
+      if (!editor) return;
+      pushUndoState();
+      var esc = function(t) { return t.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;'); };
+      editor.innerHTML = esc(_dialectResult);
+      editor.dispatchEvent(new Event('input', { bubbles: true }));
+      switchTab('write');
+      if (typeof showToast === 'function') showToast('✓ تم تطبيق النص الفصيح في المحرر');
+    }
 
     /* ═══════════════════════════════════════════
        Quran Verification & Translation

src/nlp/dialect/__init__.py

@@ -0,0 +1 @@
+# Dialect-to-MSA conversion module

src/nlp/dialect/dialect_service.py

@@ -0,0 +1,80 @@
+"""
+Dialect-to-MSA (Modern Standard Arabic) conversion service.
+
+Uses bayan10/dialect-to-msa-model (mT5 300M) to convert colloquial
+Arabic dialects (Egyptian, Gulf, Levantine, Maghrebi) to formal MSA.
+
+Singleton pattern — lazy-loads the model on first request to avoid
+blocking server startup.
+"""
+
+import logging
+import torch
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+
+logger = logging.getLogger(__name__)
+
+_instance = None
+
+
+class DialectConverter:
+    """Converts dialect Arabic text to Modern Standard Arabic (MSA)."""
+
+    PREFIX = "حوّل إلى الفصحى: "
+    REPO_ID = "bayan10/dialect-to-msa-model"
+    MAX_INPUT_LENGTH = 128
+    MAX_OUTPUT_LENGTH = 128
+
+    def __init__(self):
+        self.device = "cpu"
+        logger.info(f"[DIALECT] Loading model from '{self.REPO_ID}'...")
+
+        self.tokenizer = AutoTokenizer.from_pretrained(self.REPO_ID)
+        self.model = AutoModelForSeq2SeqLM.from_pretrained(
+            self.REPO_ID, torch_dtype=torch.float16
+        ).to(self.device)
+        self.model.eval()
+
+        logger.info("[DIALECT] Model loaded successfully (float16).")
+
+    def convert(self, dialect_text: str, num_beams: int = 4) -> str:
+        """Convert a single dialect sentence to MSA."""
+        if not dialect_text or not dialect_text.strip():
+            return dialect_text
+
+        input_text = self.PREFIX + dialect_text.strip()
+        inputs = self.tokenizer(
+            input_text,
+            return_tensors="pt",
+            max_length=self.MAX_INPUT_LENGTH,
+            truncation=True,
+        ).to(self.device)
+
+        with torch.no_grad():
+            outputs = self.model.generate(
+                **inputs,
+                max_length=self.MAX_OUTPUT_LENGTH,
+                num_beams=num_beams,
+                early_stopping=True,
+                no_repeat_ngram_size=3,
+            )
+
+        result = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+        return result
+
+    def is_ready(self) -> bool:
+        """Check if the model is loaded and ready."""
+        return self.model is not None and self.tokenizer is not None
+
+
+def get_dialect_model() -> DialectConverter:
+    """Get or create the singleton DialectConverter instance."""
+    global _instance
+    if _instance is None:
+        _instance = DialectConverter()
+    return _instance
+
+
+def is_loaded() -> bool:
+    """Check if the dialect model is loaded (without triggering lazy load)."""
+    return _instance is not None and _instance.is_ready()