feat(NLP-4): AutoComplete integration — hybrid bigram + GPT-2

COMPLETELY INDEPENDENT from the correction pipeline. Zero interaction
with /api/analyze, StageLocker, OffsetMapper, PatchSet, or highlights.

Backend:
- src/nlp/autocomplete/ package (service, rules, init)
- HybridAutoComplete: lazy-loaded singleton, LRU cache, OOM fallback
- Bigram model from bayan10/AutoComplete (bigram_model_v4.pkl)
- GPT-2 from aubmindlab/aragpt2-base (optional, falls back to bigram)
- POST /api/autocomplete with {context, n} → {status, suggestions}
- /api/health now reports autocomplete status

Frontend:
- src/js/autocomplete.js — self-contained module
- Ghost text overlay (gray, semi-transparent, pointer-events:none)
- Dropdown with keyboard nav (↑/↓), TAB accept, ESC dismiss
- 400ms debounce, min 3 chars, cursor-aware context extraction
- CSS in components.css with dark/light theme support

59/59 existing pipeline tests passing

src/app.py

@@ -158,7 +158,7 @@ def health_check():
             'models': {
                 'summarization': summarization_model is not None,
                 'spelling': _spelling_available(),
-                'autocomplete': False,
+                'autocomplete': _autocomplete_available(),
                 'grammar': _grammar_available(),
                 'punctuation': _punctuation_available()
             },
@@ -254,6 +254,15 @@ def _punctuation_available():
         return False
 
 
+def _autocomplete_available():
+    """Check if autocomplete model is loaded (without triggering lazy load)."""
+    try:
+        from nlp.autocomplete.autocomplete_service import _instance
+        return _instance is not None and _instance.is_ready()
+    except Exception:
+        return False
+
+
 @app.route('/api/spelling', methods=['POST'])
 def spelling_correction():
     """
@@ -409,50 +418,59 @@ def summarize():
 def autocomplete():
     """
     Get autocomplete suggestions for Arabic text.
-    
-    Expected JSON payload:
+    COMPLETELY INDEPENDENT — has zero interaction with /api/analyze.
+
+    Request JSON:
+    {
+        "context": "<text before cursor>",
+        "n": 5 (optional)
+    }
+
+    Response JSON:
     {
-        "text": "Arabic text prefix",
-        "n": 5 (number of suggestions, optional)
+        "status": "success",
+        "suggestions": ["word1", "word2", ...]
     }
     """
-    if not USE_HF_API and autocomplete_model is None:
-        return jsonify({
-            'error': 'Autocomplete model not loaded. Please check server logs.',
-            'status': 'error'
-        }), 503
-    
     try:
         if not request.is_json:
             return jsonify({'error': 'Request must be JSON', 'status': 'error'}), 400
-        
+
         data = request.get_json()
-        text = data.get('text', '').strip()
+        context = data.get('context', '').strip()
         n = int(data.get('n', 5))
-        
-        if not text:
-            return jsonify({'error': 'Text is required', 'status': 'error'}), 400
-        
-        logger.info(f"Getting autocomplete suggestions for: {text[:50]}...")
-        if USE_HF_API:
-            suggestions = hf_autocomplete(text, n=n)
-        else:
-            suggestions = autocomplete_model.predict(text, n=n)
-        logger.info(f"Autocomplete suggestions (n={n}): {suggestions}")
-        
+
+        if not context or len(context) < 3:
+            return jsonify({'suggestions': [], 'status': 'success'})
+
+        # Extract last ~200 chars (trimmed to word boundary)
+        from nlp.autocomplete.autocomplete_rules import extract_context
+        context = extract_context(context, max_chars=200)
+
+        # Lazy-load the model on first request
+        from nlp.autocomplete.autocomplete_service import get_autocomplete_model
+        ac_model = get_autocomplete_model()
+
+        if not ac_model.is_ready():
+            return jsonify({'suggestions': [], 'status': 'success'})
+
+        t0 = time.time()
+        suggestions = ac_model.predict(context, n=n)
+        elapsed = int((time.time() - t0) * 1000)
+        logger.info(f"[AUTOCOMPLETE] {elapsed}ms | mode={ac_model.get_mode()} | suggestions={suggestions}")
+
         return jsonify({
             'suggestions': suggestions,
             'status': 'success'
         })
-    
+
     except Exception as e:
         logger.error(f"Error during autocomplete: {str(e)}")
         logger.error(traceback.format_exc())
         return jsonify({
-            'error': 'An error occurred during autocomplete.',
-            'status': 'error',
-            'details': str(e) if app.debug else None
-        }), 500
+            'suggestions': [],
+            'status': 'success'  # Graceful degradation — never fail the UI
+        })
 
 
 @app.route('/api/grammar', methods=['POST'])

src/css/components.css

@@ -2794,3 +2794,119 @@ input[type="range"]::-moz-range-thumb {
   box-shadow: 0 4px 16px rgba(26, 29, 33, 0.15);
 }
 
+/* ═══════════════════════════════════════════════════════════════════
+   NLP-4: AutoComplete — Ghost Text + Dropdown
+   COMPLETELY INDEPENDENT from correction pipeline highlights.
+   ═══════════════════════════════════════════════════════════════════ */
+
+/* ── Ghost Text Overlay ── */
+#autocomplete-ghost {
+  position: absolute;
+  pointer-events: none;
+  color: var(--color-text-muted, #6b7280);
+  opacity: 0.45;
+  font-family: inherit;
+  font-size: inherit;
+  line-height: inherit;
+  z-index: 5;
+  white-space: pre;
+  direction: rtl;
+  display: none;
+  user-select: none;
+  -webkit-user-select: none;
+}
+
+/* ── Dropdown ── */
+#autocomplete-dropdown {
+  position: fixed;
+  background: var(--color-surface, #1a1d21);
+  border: 1px solid var(--color-border, rgba(255,255,255,0.1));
+  border-radius: 10px;
+  box-shadow:
+    0 4px 24px rgba(0, 0, 0, 0.25),
+    0 0 0 1px rgba(255, 255, 255, 0.05) inset;
+  z-index: 9999;
+  max-height: 220px;
+  overflow-y: auto;
+  direction: rtl;
+  min-width: 160px;
+  padding: 4px;
+  backdrop-filter: blur(12px);
+  -webkit-backdrop-filter: blur(12px);
+  animation: ac-dropdown-in 0.15s ease;
+}
+
+@keyframes ac-dropdown-in {
+  from { opacity: 0; transform: translateY(-4px); }
+  to   { opacity: 1; transform: translateY(0); }
+}
+
+/* ── Dropdown Items ── */
+.ac-dropdown-item {
+  padding: 8px 14px;
+  font-size: 0.95rem;
+  color: var(--color-text-primary, #e5e7eb);
+  cursor: pointer;
+  border-radius: 7px;
+  transition: background 0.12s ease, color 0.12s ease;
+  direction: rtl;
+  text-align: right;
+  font-family: inherit;
+  line-height: 1.5;
+}
+
+.ac-dropdown-item:hover,
+.ac-dropdown-item.ac-selected {
+  background: linear-gradient(135deg,
+    rgba(107, 163, 224, 0.15),
+    rgba(160, 131, 237, 0.12));
+  color: var(--color-primary, #6ba3e0);
+}
+
+.ac-dropdown-item.ac-selected {
+  font-weight: 600;
+}
+
+/* ── Scrollbar ── */
+#autocomplete-dropdown::-webkit-scrollbar {
+  width: 4px;
+}
+#autocomplete-dropdown::-webkit-scrollbar-track {
+  background: transparent;
+}
+#autocomplete-dropdown::-webkit-scrollbar-thumb {
+  background: rgba(255, 255, 255, 0.12);
+  border-radius: 2px;
+}
+
+/* ── Light theme overrides ── */
+[data-theme="light"] #autocomplete-ghost {
+  color: #9ca3af;
+  opacity: 0.5;
+}
+
+[data-theme="light"] #autocomplete-dropdown {
+  background: #ffffff;
+  border-color: rgba(0, 0, 0, 0.1);
+  box-shadow:
+    0 4px 24px rgba(0, 0, 0, 0.12),
+    0 0 0 1px rgba(0, 0, 0, 0.04) inset;
+}
+
+[data-theme="light"] .ac-dropdown-item {
+  color: #374151;
+}
+
+[data-theme="light"] .ac-dropdown-item:hover,
+[data-theme="light"] .ac-dropdown-item.ac-selected {
+  background: rgba(107, 163, 224, 0.1);
+  color: #2563eb;
+}
+
+/* ── Print: hide autocomplete ── */
+@media print {
+  #autocomplete-ghost,
+  #autocomplete-dropdown {
+    display: none !important;
+  }
+}

src/index.html

@@ -32,6 +32,7 @@
   <script src="/js/ui.js"></script>
   <script src="/js/documents/doc-utils.js"></script>
   <script src="/js/editor.js"></script>
+  <script src="/js/autocomplete.js"></script>
   <script src="/js/format.js"></script>
   <script src="/js/documents/import.js"></script>
   <script src="/js/documents/export.js?v=20260615d"></script>

src/js/autocomplete.js

@@ -0,0 +1,425 @@
+/**
+ * AutoComplete Module — Ghost Text + Dropdown for Arabic autocomplete.
+ *
+ * COMPLETELY INDEPENDENT from the correction pipeline.
+ * This module has ZERO interaction with:
+ * - editor.js correction/highlight logic
+ * - renderer.js span rendering
+ * - ui.js suggestion sidebar
+ * - /api/analyze
+ *
+ * It only talks to: /api/autocomplete (its own endpoint)
+ */
+
+(function () {
+  'use strict';
+
+  // ─── Configuration ───────────────────────────────────────────────
+  const DEBOUNCE_MS = 400;
+  const MIN_CONTEXT_LEN = 3;
+  const MAX_SUGGESTIONS = 5;
+  const CONTEXT_CHARS = 200;
+
+  // ─── State ───────────────────────────────────────────────────────
+  let ghostEl = null;
+  let dropdownEl = null;
+  let selectedIndex = -1;
+  let currentSuggestions = [];
+  let debounceTimer = null;
+  let isComposing = false;
+  let editorEl = null;
+
+  // ─── Initialization ──────────────────────────────────────────────
+  function init() {
+    editorEl = document.getElementById('editor-container');
+    if (!editorEl) {
+      // Retry after DOM is ready
+      setTimeout(init, 500);
+      return;
+    }
+
+    createGhostElement();
+    createDropdownElement();
+    bindEvents();
+    console.log('[AutoComplete] Initialized');
+  }
+
+  // ─── Ghost Text Element ──────────────────────────────────────────
+  function createGhostElement() {
+    ghostEl = document.createElement('div');
+    ghostEl.id = 'autocomplete-ghost';
+    ghostEl.setAttribute('aria-hidden', 'true');
+    // Position relative to editor's parent
+    const editorParent = editorEl.parentElement;
+    if (editorParent) {
+      editorParent.style.position = 'relative';
+      editorParent.appendChild(ghostEl);
+    }
+  }
+
+  // ─── Dropdown Element ────────────────────────────────────────────
+  function createDropdownElement() {
+    dropdownEl = document.createElement('div');
+    dropdownEl.id = 'autocomplete-dropdown';
+    dropdownEl.setAttribute('role', 'listbox');
+    dropdownEl.setAttribute('aria-label', 'اقتراحات الإكمال التلقائي');
+    dropdownEl.style.display = 'none';
+    document.body.appendChild(dropdownEl);
+  }
+
+  // ─── Event Binding ───────────────────────────────────────────────
+  function bindEvents() {
+    // Typing → debounced autocomplete
+    editorEl.addEventListener('input', onInput);
+
+    // Composition events (IME)
+    editorEl.addEventListener('compositionstart', () => { isComposing = true; });
+    editorEl.addEventListener('compositionend', () => { isComposing = false; });
+
+    // Keyboard: TAB accept, ESC dismiss, arrow navigation
+    editorEl.addEventListener('keydown', onKeyDown);
+
+    // Cursor movement / selection change → dismiss
+    document.addEventListener('selectionchange', onSelectionChange);
+
+    // Click outside → dismiss
+    document.addEventListener('mousedown', function (e) {
+      if (dropdownEl && !dropdownEl.contains(e.target) && e.target !== editorEl) {
+        dismiss();
+      }
+    });
+
+    // Scroll → reposition
+    editorEl.addEventListener('scroll', dismiss);
+
+    // Window resize → dismiss
+    window.addEventListener('resize', dismiss);
+  }
+
+  // ─── Input Handler ───────────────────────────────────────────────
+  function onInput() {
+    if (isComposing) return;
+
+    clearTimeout(debounceTimer);
+    debounceTimer = setTimeout(fetchSuggestions, DEBOUNCE_MS);
+  }
+
+  // ─── Selection Change → Dismiss ──────────────────────────────────
+  function onSelectionChange() {
+    const sel = window.getSelection();
+    if (!sel || !sel.isCollapsed) {
+      dismiss();
+      return;
+    }
+    // If cursor moved (not from typing), dismiss
+    // We rely on the debounce to re-trigger if user is still typing
+  }
+
+  // ─── Keyboard Handler ───────────────────────────────────────────
+  function onKeyDown(e) {
+    if (!isVisible()) return;
+
+    switch (e.key) {
+      case 'Tab':
+        e.preventDefault();
+        acceptSuggestion();
+        break;
+
+      case 'Escape':
+        e.preventDefault();
+        dismiss();
+        break;
+
+      case 'ArrowDown':
+        e.preventDefault();
+        navigateDropdown(1);
+        break;
+
+      case 'ArrowUp':
+        e.preventDefault();
+        navigateDropdown(-1);
+        break;
+
+      default:
+        // Any other key → will trigger onInput → new debounce
+        // Dismiss current ghost immediately for responsiveness
+        hideGhost();
+        break;
+    }
+  }
+
+  // ─── Fetch Suggestions ───────────────────────────────────────────
+  async function fetchSuggestions() {
+    const sel = window.getSelection();
+    if (!sel || !sel.isCollapsed || !sel.rangeCount) {
+      dismiss();
+      return;
+    }
+
+    // Check the editor has text
+    const text = editorEl.innerText || editorEl.textContent || '';
+    if (text.trim().length < MIN_CONTEXT_LEN) {
+      dismiss();
+      return;
+    }
+
+    // Extract context: text before cursor (last N chars)
+    const context = getTextBeforeCursor(CONTEXT_CHARS);
+    if (!context || context.trim().length < MIN_CONTEXT_LEN) {
+      dismiss();
+      return;
+    }
+
+    try {
+      const resp = await fetch('/api/autocomplete', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          context: context,
+          n: MAX_SUGGESTIONS
+        })
+      });
+
+      if (!resp.ok) {
+        dismiss();
+        return;
+      }
+
+      const data = await resp.json();
+      if (data.status !== 'success' || !data.suggestions || !data.suggestions.length) {
+        dismiss();
+        return;
+      }
+
+      showSuggestions(data.suggestions);
+
+    } catch (err) {
+      console.warn('[AutoComplete] Fetch error:', err);
+      dismiss();
+    }
+  }
+
+  // ─── Get Text Before Cursor ──────────────────────────────────────
+  function getTextBeforeCursor(maxChars) {
+    const sel = window.getSelection();
+    if (!sel || !sel.rangeCount) return '';
+
+    try {
+      const range = sel.getRangeAt(0);
+      const preRange = document.createRange();
+      preRange.selectNodeContents(editorEl);
+      preRange.setEnd(range.startContainer, range.startOffset);
+      const text = preRange.toString();
+      preRange.detach();
+
+      if (text.length <= maxChars) return text;
+      return text.slice(-maxChars);
+    } catch (e) {
+      return '';
+    }
+  }
+
+  // ─── Show Suggestions ────────────────────────────────────────────
+  function showSuggestions(suggestions) {
+    currentSuggestions = suggestions;
+    selectedIndex = 0; // Pre-select first
+
+    // Show ghost text (best suggestion)
+    showGhost(suggestions[0]);
+
+    // Build dropdown
+    dropdownEl.innerHTML = '';
+    suggestions.forEach(function (word, idx) {
+      const item = document.createElement('div');
+      item.className = 'ac-dropdown-item' + (idx === 0 ? ' ac-selected' : '');
+      item.setAttribute('role', 'option');
+      item.textContent = word;
+      item.addEventListener('mousedown', function (e) {
+        e.preventDefault();
+        selectedIndex = idx;
+        acceptSuggestion();
+      });
+      item.addEventListener('mouseenter', function () {
+        selectedIndex = idx;
+        updateDropdownSelection();
+      });
+      dropdownEl.appendChild(item);
+    });
+
+    // Position dropdown near caret
+    positionDropdown();
+    dropdownEl.style.display = 'block';
+  }
+
+  // ─── Ghost Text ──────────────────────────────────────────────────
+  function showGhost(text) {
+    if (!ghostEl || !text) return;
+
+    // Get caret position relative to editor
+    const caretPos = getCaretCoordinates();
+    if (!caretPos) {
+      hideGhost();
+      return;
+    }
+
+    ghostEl.textContent = text;
+    ghostEl.style.display = 'block';
+
+    // Position ghost at caret
+    const editorRect = editorEl.getBoundingClientRect();
+    const parentRect = editorEl.parentElement.getBoundingClientRect();
+
+    // RTL: ghost appears to the LEFT of the caret
+    ghostEl.style.top = (caretPos.top - parentRect.top) + 'px';
+    ghostEl.style.right = (parentRect.right - caretPos.left + 4) + 'px';
+    ghostEl.style.left = 'auto';
+  }
+
+  function hideGhost() {
+    if (ghostEl) {
+      ghostEl.style.display = 'none';
+      ghostEl.textContent = '';
+    }
+  }
+
+  // ─── Dropdown Position ───────────────────────────────────────────
+  function positionDropdown() {
+    const caretPos = getCaretCoordinates();
+    if (!caretPos) return;
+
+    const lineHeight = parseInt(getComputedStyle(editorEl).lineHeight) || 24;
+
+    // Position below caret
+    dropdownEl.style.position = 'fixed';
+    dropdownEl.style.top = (caretPos.bottom + 4) + 'px';
+
+    // RTL: align to the right of caret
+    dropdownEl.style.right = (window.innerWidth - caretPos.left) + 'px';
+    dropdownEl.style.left = 'auto';
+
+    // Ensure dropdown doesn't go off-screen
+    const rect = dropdownEl.getBoundingClientRect();
+    if (rect.bottom > window.innerHeight - 20) {
+      // Show above caret instead
+      dropdownEl.style.top = (caretPos.top - rect.height - 4) + 'px';
+    }
+  }
+
+  // ─── Dropdown Navigation ─────────────────────────────────────────
+  function navigateDropdown(direction) {
+    if (!currentSuggestions.length) return;
+
+    selectedIndex += direction;
+    if (selectedIndex < 0) selectedIndex = currentSuggestions.length - 1;
+    if (selectedIndex >= currentSuggestions.length) selectedIndex = 0;
+
+    updateDropdownSelection();
+    showGhost(currentSuggestions[selectedIndex]);
+  }
+
+  function updateDropdownSelection() {
+    const items = dropdownEl.querySelectorAll('.ac-dropdown-item');
+    items.forEach(function (item, idx) {
+      item.classList.toggle('ac-selected', idx === selectedIndex);
+    });
+
+    // Scroll selected item into view
+    const selected = dropdownEl.querySelector('.ac-selected');
+    if (selected) {
+      selected.scrollIntoView({ block: 'nearest' });
+    }
+  }
+
+  // ─── Accept Suggestion ───────────────────────────────────────────
+  function acceptSuggestion() {
+    if (selectedIndex < 0 || selectedIndex >= currentSuggestions.length) {
+      dismiss();
+      return;
+    }
+
+    const word = currentSuggestions[selectedIndex];
+
+    // Insert the word at cursor position
+    const sel = window.getSelection();
+    if (!sel || !sel.rangeCount) {
+      dismiss();
+      return;
+    }
+
+    // Insert with a space before the word
+    const textToInsert = word + ' ';
+    const range = sel.getRangeAt(0);
+    range.deleteContents();
+    const textNode = document.createTextNode(textToInsert);
+    range.insertNode(textNode);
+
+    // Move caret to end of inserted text
+    range.setStartAfter(textNode);
+    range.setEndAfter(textNode);
+    sel.removeAllRanges();
+    sel.addRange(range);
+
+    dismiss();
+
+    // Trigger input event so the editor knows text changed
+    editorEl.dispatchEvent(new Event('input', { bubbles: true }));
+  }
+
+  // ─── Dismiss ─────────────────────────────────────────────────────
+  function dismiss() {
+    hideGhost();
+    currentSuggestions = [];
+    selectedIndex = -1;
+    if (dropdownEl) {
+      dropdownEl.style.display = 'none';
+      dropdownEl.innerHTML = '';
+    }
+  }
+
+  // ─── Helpers ─────────────────────────────────────────────────────
+  function isVisible() {
+    return dropdownEl && dropdownEl.style.display !== 'none';
+  }
+
+  function getCaretCoordinates() {
+    const sel = window.getSelection();
+    if (!sel || !sel.rangeCount) return null;
+
+    try {
+      const range = sel.getRangeAt(0).cloneRange();
+      range.collapse(true);
+
+      // Use a zero-width space to get coordinates
+      const span = document.createElement('span');
+      span.textContent = '\u200B';
+      range.insertNode(span);
+
+      const rect = span.getBoundingClientRect();
+      const coords = {
+        top: rect.top,
+        left: rect.left,
+        bottom: rect.bottom,
+        right: rect.right
+      };
+
+      // Clean up
+      span.parentNode.removeChild(span);
+
+      // Restore selection
+      sel.removeAllRanges();
+      sel.addRange(range);
+
+      return coords;
+    } catch (e) {
+      return null;
+    }
+  }
+
+  // ─── Initialize on DOM ready ─────────────────────────────────────
+  if (document.readyState === 'loading') {
+    document.addEventListener('DOMContentLoaded', init);
+  } else {
+    init();
+  }
+
+})();

src/nlp/autocomplete/__init__.py

@@ -0,0 +1 @@
+# AutoComplete package — completely independent from the correction pipeline

src/nlp/autocomplete/autocomplete_rules.py

@@ -0,0 +1,86 @@
+"""
+AutoComplete Rules — text processing utilities for Arabic autocomplete.
+Completely independent from the correction pipeline.
+"""
+
+import re
+from collections import defaultdict
+
+
+def canonical_form(word: str) -> str:
+    """
+    Normalize Arabic word to a canonical form for deduplication.
+    Collapses hamza variants, ta marbuta, alef maqsura, and diacritics.
+    """
+    word = re.sub("[إأآا]", "ا", word)
+    word = re.sub("ى", "ي", word)
+    word = re.sub("ؤ", "و", word)
+    word = re.sub("ئ", "ي", word)
+    word = re.sub("ة", "ه", word)
+    word = re.sub(r"[ًٌٍَُِّْ]", "", word)
+    return word
+
+
+def merge_similar_predictions(preds: list, top_k: int = 20) -> list:
+    """
+    Merge predictions that differ only in diacritics/hamza variants.
+    Groups by canonical form, sums scores, keeps the first surface form.
+    """
+    groups = defaultdict(lambda: {"score": 0.0, "words": []})
+
+    for w, p in preds:
+        key = canonical_form(w)
+        groups[key]["score"] += p
+        groups[key]["words"].append(w)
+
+    merged = sorted(
+        groups.values(),
+        key=lambda x: x["score"],
+        reverse=True
+    )
+
+    return [
+        (group["words"][0], group["score"])
+        for group in merged[:top_k]
+    ]
+
+
+def filter_suggestions(suggestions: list, min_len: int = 2, max_len: int = 30) -> list:
+    """
+    Filter autocomplete suggestions:
+    - Remove too-short or too-long words
+    - Remove non-Arabic words
+    - Remove punctuation-only tokens
+    """
+    ARABIC_RE = re.compile(r'[\u0600-\u06FF]')
+    filtered = []
+    for word, score in suggestions:
+        word = word.strip()
+        if not word or len(word) < min_len or len(word) > max_len:
+            continue
+        if not ARABIC_RE.search(word):
+            continue
+        # Skip punctuation-only or whitespace-only
+        if all(c in '.,;:!?،؛؟!.:«»"\'()-–—… \t\n' for c in word):
+            continue
+        filtered.append((word, score))
+    return filtered
+
+
+def extract_context(text: str, max_chars: int = 200) -> str:
+    """
+    Extract the last N characters of text, trimmed to a word boundary.
+    This is the context sent to the autocomplete model.
+    """
+    if not text or len(text) <= max_chars:
+        return text.strip()
+
+    # Take last max_chars characters
+    snippet = text[-max_chars:]
+
+    # Trim to word boundary (don't send a partial word at the start)
+    first_space = snippet.find(' ')
+    if first_space > 0 and first_space < len(snippet) // 2:
+        snippet = snippet[first_space + 1:]
+
+    return snippet.strip()

src/nlp/autocomplete/autocomplete_service.py

@@ -0,0 +1,307 @@
+"""
+AutoComplete Service — Hybrid bigram + GPT-2 Arabic autocomplete.
+
+COMPLETELY INDEPENDENT from the correction pipeline.
+This module has ZERO interaction with:
+- /api/analyze
+- StageLockManager / OffsetMapper / ClaimedRanges
+- OverlapResolver / PatchSet / CorrectionPatch
+- Highlight rendering
+
+Architecture:
+    User types → debounce → POST /api/autocomplete
+    → HybridAutoComplete.predict(context)
+    → Bigram lookup + GPT-2 scoring
+    → Ranked suggestions returned to frontend
+"""
+
+import os
+import time
+import pickle
+import logging
+import threading
+from functools import lru_cache
+
+import torch
+from huggingface_hub import hf_hub_download
+
+logger = logging.getLogger(__name__)
+
+# ─── Singleton ────────────────────────────────────────────────────────────────
+_instance = None
+_lock = threading.Lock()
+
+
+def get_autocomplete_model():
+    """Lazy-loaded singleton — returns the cached HybridAutoComplete instance."""
+    global _instance
+    if _instance is not None:
+        return _instance
+
+    with _lock:
+        if _instance is not None:
+            return _instance
+        _instance = HybridAutoComplete()
+    return _instance
+
+
+# ─── Cache key helper ─────────────────────────────────────────────────────────
+def _context_key(context: str) -> str:
+    """Reduce context to last 3 words for cache key."""
+    words = context.strip().split()
+    return " ".join(words[-3:]) if words else ""
+
+
+# ─── Main Service ─────────────────────────────────────────────────────────────
+class HybridAutoComplete:
+    """
+    Hybrid Arabic autocomplete:
+    1. Statistical (bigram) — fast, always available
+    2. Neural (GPT-2) — contextual, optional (may OOM on free tier)
+    3. Hybrid scoring: alpha * stat + (1-alpha) * neural
+    """
+
+    BIGRAM_REPO = "bayan10/AutoComplete"
+    BIGRAM_FILE = "bigram_model_v4.pkl"
+    GPT2_MODEL = "aubmindlab/aragpt2-base"
+
+    def __init__(self):
+        t0 = time.time()
+        logger.info("Loading AutoComplete model (lazy init)...")
+
+        self.unigrams = None
+        self.bigrams = None
+        self.gpt2_tokenizer = None
+        self.gpt2_model = None
+        self.device = "cpu"
+        self.alpha = 0.6  # Weight: 60% bigram, 40% GPT-2
+        self._cache = {}
+        self._cache_max = 256
+
+        # 1. Load bigram (required — small file)
+        self._load_bigram()
+
+        # 2. Load GPT-2 (optional — large model, may OOM)
+        self._load_gpt2()
+
+        elapsed = time.time() - t0
+        mode = "hybrid" if self.gpt2_model else "bigram-only"
+        logger.info(f"AutoComplete ready in {elapsed:.1f}s (mode: {mode})")
+
+    def _load_bigram(self):
+        """Load bigram model from HuggingFace Hub."""
+        try:
+            path = hf_hub_download(
+                repo_id=self.BIGRAM_REPO,
+                filename=self.BIGRAM_FILE,
+            )
+            with open(path, "rb") as f:
+                data = pickle.load(f)
+            self.unigrams = data["unigrams"]
+            self.bigrams = data["bigrams"]
+            logger.info(
+                f"Bigram model loaded: {len(self.unigrams)} unigrams, "
+                f"{len(self.bigrams)} bigram contexts"
+            )
+        except Exception as e:
+            logger.error(f"Failed to load bigram model: {e}")
+            self.unigrams = {}
+            self.bigrams = {}
+
+    def _load_gpt2(self):
+        """Load GPT-2 model with OOM fallback."""
+        try:
+            from transformers import GPT2LMHeadModel, GPT2Tokenizer
+
+            logger.info(f"Loading GPT-2 tokenizer: {self.GPT2_MODEL}")
+            self.gpt2_tokenizer = GPT2Tokenizer.from_pretrained(self.GPT2_MODEL)
+            self.gpt2_tokenizer.pad_token = self.gpt2_tokenizer.eos_token
+
+            logger.info(f"Loading GPT-2 model: {self.GPT2_MODEL}")
+            self.gpt2_model = GPT2LMHeadModel.from_pretrained(self.GPT2_MODEL)
+            self.gpt2_model.config.pad_token_id = self.gpt2_tokenizer.eos_token_id
+            self.gpt2_model.eval()
+
+            logger.info("GPT-2 loaded successfully (hybrid mode enabled)")
+
+        except (torch.cuda.OutOfMemoryError, MemoryError, RuntimeError) as e:
+            logger.warning(f"GPT-2 OOM — falling back to bigram-only mode: {e}")
+            self.gpt2_tokenizer = None
+            self.gpt2_model = None
+        except Exception as e:
+            logger.warning(f"GPT-2 load failed — bigram-only mode: {e}")
+            self.gpt2_tokenizer = None
+            self.gpt2_model = None
+
+    # ─── Prediction ──────────────────────────────���────────────────────────
+
+    def predict(self, context: str, n: int = 5) -> list:
+        """
+        Get top-N autocomplete suggestions for the given context.
+
+        Args:
+            context: Text before the cursor (last ~200 chars)
+            n: Number of suggestions to return
+
+        Returns:
+            List of suggestion strings (ranked by score)
+        """
+        if not context or not context.strip():
+            return []
+
+        context = context.strip()
+
+        # Check cache
+        cache_key = _context_key(context)
+        if cache_key in self._cache:
+            return self._cache[cache_key][:n]
+
+        try:
+            if self.gpt2_model is not None:
+                results = self._hybrid_predict(context, n)
+            else:
+                results = self._bigram_predict(context, n)
+
+            # Cache the result
+            if len(self._cache) >= self._cache_max:
+                # Evict oldest entries (simple FIFO)
+                keys = list(self._cache.keys())
+                for k in keys[:len(keys) // 2]:
+                    del self._cache[k]
+            self._cache[cache_key] = results
+
+            return results[:n]
+
+        except Exception as e:
+            logger.error(f"AutoComplete prediction error: {e}")
+            return []
+
+    def _bigram_predict(self, context: str, n: int = 5) -> list:
+        """Statistical-only prediction using bigram model."""
+        from .autocomplete_rules import merge_similar_predictions, filter_suggestions
+
+        tokens = context.strip().split()
+        if not tokens:
+            return []
+
+        last_word = tokens[-1]
+        candidates = []
+
+        # Try bigram lookup
+        if last_word in self.bigrams:
+            for w, c in self.bigrams[last_word].items():
+                if len(w) < 2 or w == last_word:
+                    continue
+                candidates.append((w, c))
+
+        # Fallback to unigram if no bigram matches
+        if not candidates:
+            for w, c in self.unigrams.items():
+                if len(w) < 2:
+                    continue
+                candidates.append((w, c))
+
+        if not candidates:
+            return []
+
+        total = sum(c for _, c in candidates)
+        if total == 0:
+            return []
+
+        preds = [(w, c / total) for w, c in candidates]
+        preds.sort(key=lambda x: x[1], reverse=True)
+        preds = merge_similar_predictions(preds, top_k=n * 3)
+        preds = filter_suggestions(preds)
+
+        return [w for w, _ in preds[:n]]
+
+    def _hybrid_predict(self, context: str, n: int = 5) -> list:
+        """Hybrid prediction: bigram + GPT-2 scoring."""
+        from .autocomplete_rules import merge_similar_predictions, filter_suggestions
+
+        tokens = context.strip().split()
+        if not tokens:
+            return []
+
+        last_word = tokens[-1]
+
+        # 1. Get bigram candidates
+        stat_candidates = []
+        if last_word in self.bigrams:
+            for w, c in self.bigrams[last_word].items():
+                if len(w) < 2 or w == last_word:
+                    continue
+                stat_candidates.append((w, c))
+
+        if not stat_candidates:
+            # No bigram matches — fall back to bigram-only with unigrams
+            return self._bigram_predict(context, n)
+
+        total = sum(c for _, c in stat_candidates)
+        if total == 0:
+            return self._bigram_predict(context, n)
+
+        stat_preds = [(w, c / total) for w, c in stat_candidates]
+        stat_preds.sort(key=lambda x: x[1], reverse=True)
+        stat_preds = merge_similar_predictions(stat_preds, top_k=20)
+
+        # 2. Get GPT-2 next-token probabilities (ONCE)
+        gpt2_probs = self._gpt2_next_token_probs(context, top_k=50)
+
+        # 3. Hybrid scoring
+        results = []
+        for w, stat_p in stat_preds:
+            neural_p = gpt2_probs.get(w, 1e-8)
+            score = self.alpha * stat_p + (1 - self.alpha) * neural_p
+            results.append((w, score))
+
+        results.sort(key=lambda x: x[1], reverse=True)
+        results = filter_suggestions(results)
+
+        return [w for w, _ in results[:n]]
+
+    def _gpt2_next_token_probs(self, prefix: str, top_k: int = 50) -> dict:
+        """Get GPT-2 next-token probability distribution."""
+        if self.gpt2_model is None or self.gpt2_tokenizer is None:
+            return {}
+
+        try:
+            inputs = self.gpt2_tokenizer(
+                prefix,
+                return_tensors="pt",
+                truncation=True,
+                max_length=512,  # Shorter than 1024 for speed
+            )
+
+            with torch.no_grad():
+                outputs = self.gpt2_model(**inputs)
+                logits = outputs.logits[0, -1]
+
+            probs = torch.softmax(logits, dim=-1)
+            top_probs, top_ids = torch.topk(probs, top_k)
+
+            prob_dict = {}
+            for idx, prob in zip(top_ids, top_probs):
+                word = self.gpt2_tokenizer.decode([idx]).strip()
+                if word and len(word) >= 2:
+                    prob_dict[word] = prob.item()
+
+            return prob_dict
+
+        except Exception as e:
+            logger.warning(f"GPT-2 scoring failed: {e}")
+            return {}
+
+    # ─── Health ───────────────────────────────────────────────────────────
+
+    def is_ready(self) -> bool:
+        """Returns True if at least the bigram model is loaded."""
+        return bool(self.unigrams)
+
+    def get_mode(self) -> str:
+        """Returns 'hybrid', 'bigram-only', or 'unavailable'."""
+        if self.gpt2_model and self.unigrams:
+            return "hybrid"
+        elif self.unigrams:
+            return "bigram-only"
+        return "unavailable"