+ Event: 'input' event fired
+
+ ↓
+
+STEP 2: Event Listener Trigger
+ File: src/js/editor.js
+ Function: initEditor()
+ Line: 14-16
+ Code: editor.addEventListener('input', () => {
+ analyzeTextDelayed();
+ });
+
+ ↓
+
+STEP 3: Debounced Analysis
+ File: src/js/editor.js
+ Function: analyzeTextDelayed()
+ Line: 64-67
+ Code: setTimeout(() => {
+ analyzeText();
+ }, ANALYZE_DEBOUNCE_MS); // 500ms
+
+ ↓
+
+STEP 4: Save State Before Render
+ File: src/js/editor.js
+ Function: analyzeText() → saveSelection()
+ Line: 90-91
+ Code: const savedSelection = saveSelection();
+ const currentCaretOffset = getCaretOffset();
+
+ Called from: src/js/selection.js (imported)
+
+ ↓
+
+STEP 5: Backend API Call
+ File: src/js/editor.js
+ Function: analyzeText()
+ Line: 94-99
+ Code: const response = await fetch('/api/analyze', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ text })
+ });
+
+ ↓
+
+STEP 6: Parse API Response
+ File: src/js/editor.js
+ Function: analyzeText()
+ Line: 107-110
+ Code: const data = await response.json();
+ if (data.status !== 'success' || !data.suggestions) {
+ renderWithoutSuggestions(text);
+ return;
+ }
+
+ ↓
+
+STEP 7: >>> CALL RENDERER <<<
+ File: src/js/editor.js
+ Function: analyzeText()
+ Line: 113-116
+ Code: const highlightedHtml = render({
+ text: text,
+ suggestions: data.suggestions
+ });
+
+ CALLS: render()
+ FROM: src/js/renderer.js
+ PURPOSE: Convert suggestions into safe highlighted HTML
+
+ ↓
+
+STEP 8: Inside Renderer
+ File: src/js/renderer.js
+ Function: render(input)
+ Line: 231-235
+ Code: function render(input) {
+ const { text = '', suggestions = [] } = input;
+ return renderHighlightedText(text, suggestions);
+ }
+
+ CALLS: renderHighlightedText(text, suggestions)
+
+ ↓
+
+STEP 9: Create Segments from Offsets
+ File: src/js/renderer.js
+ Function: renderHighlightedText()
+ Line: 203-210
+ Code: const segments = createSegments(text, suggestions);
+
+ createSegments() uses:
+ - sortSuggestions() [sort by offset]
+ - NO regex, NO replace()
+ - Pure offset-based slicing
+
+ ↓
+
+STEP 10: Generate Highlighted HTML
+ File: src/js/renderer.js
+ Function: renderHighlightedText()
+ Line: 211-231
+ Code: segments.forEach((segment) => {
+ if (segment.type === 'text') {
+ html += escapeHtml(segment.text);
+ } else if (segment.type === 'suggestion') {
+ const errorClass = getErrorClass(suggestion.type);
+ const escapedText = escapeHtml(segment.text);
+ html += `
${escapedText}`;
+ }
+ });
+
+ escapeHtml(): Converts <>&"' to HTML entities
+
+ ↓
+
+STEP 11: Return Safe HTML from Renderer
+ File: src/js/renderer.js
+ Line: 233
+ Return: Safe HTML string with escaped content
+
+ ↓
+
+STEP 12: Apply HTML to Editor
+ File: src/js/editor.js
+ Function: analyzeText()
+ Line: 119
+ Code: setEditorHTML(highlightedHtml);
+
+ CALLS: setEditorHTML()
+ FROM: src/js/selection.js
+ Input: Safe HTML from render()
+
+ ↓
+
+STEP 13: Set DOM Content
+ File: src/js/selection.js
+ Function: setEditorHTML(html)
+ Line: 198-201
+ Code: function setEditorHTML(html) {
+ const editor = document.getElementById('editor-container');
+ if (!editor) return;
+ editor.innerHTML = html; // Receives SAFE HTML only
+ }
+
+ ↓
+
+STEP 14: Restore Cursor/Selection
+ File: src/js/editor.js
+ Function: analyzeText()
+ Line: 122-126
+ Code: if (savedSelection) {
+ restoreSelection(savedSelection);
+ } else {
+ setCaretOffset(currentCaretOffset);
+ }
+
+ CALLS: restoreSelection() and setCaretOffset()
+ FROM: src/js/selection.js
+ Effect: Cursor/selection returned to original position
+
+ ↓
+
+STEP 15: Update UI Counters
+ File: src/js/editor.js
+ Function: analyzeText()
+ Line: 128-132
+ Code: const spellingCount = data.suggestions.filter(...).length;
+ const grammarCount = data.suggestions.filter(...).length;
+ const punctuationCount = data.suggestions.filter(...).length;
+ updateSuggestionCounts(spelling, grammar, punctuation);
+
+ ↓
+
+COMPLETE: User sees highlighted text with cursor preserved ✅
+```
+
+### Summary of Execution Chain
+```
+User Input
+ ↓ [editor.js]
+editor.addEventListener('input')
+ ↓ [editor.js:14-16]
+analyzeTextDelayed()
+ ↓ [editor.js:64-67]
+setTimeout (500ms debounce)
+ ↓ [editor.js:70]
+analyzeText()
+ ↓ [editor.js:90-91]
+saveSelection() + getCaretOffset()
+ ↓ [editor.js:94-99]
+fetch('/api/analyze', {text})
+ ↓ [editor.js:107-110]
+response.json()
+ ↓ [editor.js:113-116]
+>>> render({text, suggestions}) <<<
+ ↓ [renderer.js:113]
+renderHighlightedText(text, suggestions)
+ ↓ [renderer.js:203-210]
+createSegments(text, suggestions)
+ ↓ [renderer.js:211-231]
+Generate safe HTML with escapeHtml()
+ ↓ [renderer.js:233]
+Return safe HTML
+ ↓ [editor.js:119]
+setEditorHTML(highlightedHtml)
+ ↓ [selection.js:201]
+editor.innerHTML = html (SAFE)
+ ↓ [editor.js:122-126]
+restoreSelection()
+ ↓ [selection.js:130-180]
+Restore cursor position
+ ↓
+Display update complete ✅
+```
+
+**Key Points**:
+- ✅ No regex matching
+- ✅ No text replacement
+- ✅ Offset-based only
+- ✅ HTML escaped
+- ✅ Cursor preserved
+- ✅ Clear data flow
+
+---
+
+## Verification 4: Proof of Import and Execution
+
+### Script Imports in index.html
+
+```html
+File: src/index.html
+Lines: 109-113
+
+
+
+
+
+```
+
+✅ **renderer.js is explicitly imported**
+
+### Function Call Sites
+
+#### Call Site 1: Direct Function Call
+```
+File: src/js/editor.js
+Line: 113
+Code: const highlightedHtml = render({
+ text: text,
+ suggestions: data.suggestions
+ });
+
+Type: Direct function call
+Function: render()
+Source: src/js/renderer.js (imported globally)
+```
+
+#### Call Site 2: Render Without Suggestions
+```
+File: src/js/editor.js
+Line: 83
+Code: renderWithoutSuggestions(text);
+
+Type: Direct function call
+Function: renderWithoutSuggestions()
+Source: src/js/editor.js (local)
+Note: Falls back to plain text when no suggestions
+```
+
+### Execution Chain for renderer.js
+
+```
+1. Script Load:
+ Status: ✅ Loads before index.js scripts
+
+2. Global Scope: All functions available globally
+ - render()
+ - renderHighlightedText()
+ - createSegments()
+ - escapeHtml()
+ - getErrorClass()
+ - sortSuggestions()
+
+3. Runtime Invocation Path:
+ initEditor()
+ ↓
+ editor.addEventListener('input', () => analyzeTextDelayed())
+ ↓
+ [500ms wait]
+ ↓
+ analyzeText()
+ ↓
+ render({text, suggestions}) ← RENDERER.JS CALLED
+ ↓
+ renderHighlightedText()
+ ↓
+ [Returns safe HTML]
+ ↓
+ setEditorHTML()
+ ↓
+ editor.innerHTML = html
+
+4. Verification: render() is called 15 lines after getting suggestions
+ File: src/js/editor.js:113
+ Code: const highlightedHtml = render({...});
+```
+
+### Module Dependency Graph
+
+```
+index.html
+├── renderer.js ✅ FIRST (line 110)
+├── selection.js ✅ SECOND (line 111)
+├── editor.js ✅ THIRD (line 112)
+│ └── Calls render() from renderer.js
+│ └── Calls saveSelection() from selection.js
+│ └── Calls setEditorHTML() from selection.js
+│ └── Calls restoreSelection() from selection.js
+│
+└── api.js (line 113)
+
+Load Order: renderer → selection → editor → api
+Dependency: editor depends on renderer and selection
+Status: ✅ Correct order enforced
+```
+
+### Proof of Execution Chain
+
+**Checkpoint 1**: renderer.js loaded globally
+```javascript
+// renderer.js defines:
+function render(input) {
+ const { text = '', suggestions = [] } = input;
+ return renderHighlightedText(text, suggestions);
+}
+```
+✅ Available globally after page load
+
+**Checkpoint 2**: editor.js calls render()
+```javascript
+// editor.js line 113:
+const highlightedHtml = render({
+ text: text,
+ suggestions: data.suggestions
+});
+```
+✅ Calls render() from renderer.js
+
+**Checkpoint 3**: Returned HTML is safe
+```javascript
+// renderer.js renderHighlightedText():
+html += escapeHtml(segment.text); // XSS-safe
+```
+✅ HTML is escaped before use
+
+**Checkpoint 4**: HTML applied via approved path
+```javascript
+// editor.js line 119:
+setEditorHTML(highlightedHtml);
+
+// selection.js line 201:
+function setEditorHTML(html) {
+ editor.innerHTML = html; // Only way innerHTML is modified
+}
+```
+✅ Single controlled path
+
+---
+
+## Summary: Verification 4
+
+✅ Imports: ✅ All 3 modules imported in correct order
+✅ Call Sites: ✅ render() called at line 113 of editor.js
+✅ Execution Chain: ✅ User input → analyzeText() → render() → setEditorHTML()
+✅ No Alternatives: ✅ No other way to highlight text exists
+✅ Proof: ✅ render() is only highlight function called
diff --git a/atef_prompt.md b/atef_prompt.md
new file mode 100644
index 0000000000000000000000000000000000000000..a0315588b2176a48025351a086f4378c0eb57454
--- /dev/null
+++ b/atef_prompt.md
@@ -0,0 +1,198 @@
+\# Agent Operating Instructions
+
+
+
+\---
+
+
+
+\## 1. Think Before Coding
+
+Don't assume. Surface tradeoffs. Name confusion early.
+
+
+
+\- \*\*State assumptions explicitly\*\* — if uncertain, ask rather than guess
+
+\- \*\*Present multiple interpretations\*\* — don't pick silently when ambiguity exists
+
+\- \*\*Push back when warranted\*\* — if a simpler approach exists, say so
+
+\- \*\*Stop when confused\*\* — name what's unclear before writing a single line
+
+
+
+\---
+
+
+
+\## 2. Simplicity First
+
+Minimum code that solves the problem. Nothing speculative.
+
+
+
+\- No features beyond what was asked
+
+\- No abstractions for single-use code
+
+\- No "flexibility" that wasn't requested
+
+\- No error handling for impossible scenarios
+
+\- If 200 lines could be 50, rewrite it
+
+
+
+\*\*Test:\*\* Would a senior engineer call this overcomplicated? If yes, simplify.
+
+
+
+\---
+
+
+
+\## 3. Surgical Changes
+
+Touch only what you must. Clean up only your own mess.
+
+
+
+When editing existing code:
+
+\- Don't "improve" adjacent code, comments, or formatting
+
+\- Don't refactor things that aren't broken
+
+\- Match existing style, even if you'd do it differently
+
+\- If you notice unrelated dead code, \*\*mention it — don't delete it\*\*
+
+
+
+When your changes create orphans:
+
+\- Remove imports/variables/functions that \*\*your\*\* changes made unused
+
+\- Leave pre-existing dead code unless explicitly asked to remove it
+
+
+
+\*\*Test:\*\* Every changed line should trace directly to the user's request.
+
+
+
+\---
+
+
+
+\## 4. Goal-Driven Execution
+
+Define success criteria. Loop until verified. Never mark done without proof.
+
+
+
+Transform vague tasks into verifiable goals:
+
+
+
+| Instead of... | Transform to... |
+
+|----------------------|------------------------------------------------------|
+
+| "Add validation" | "Write tests for invalid inputs, then make them pass"|
+
+| "Fix the bug" | "Write a test that reproduces it, then make it pass" |
+
+| "Refactor X" | "Ensure tests pass before and after" |
+
+
+
+For multi-step tasks, state a brief plan upfront:
+
+1. \[Step] → verify: \[check]
+
+2\. \[Step] → verify: \[check]
+
+3\. \[Step] → verify: \[check]
+
+\*\*Test:\*\* Would a staff engineer approve this before you mark it done?
+
+
+
+\---
+
+
+
+\## 5. Subagent Strategy
+
+Use subagents to keep the main context window clean.
+
+
+
+\- Offload research, exploration, and parallel analysis to subagents
+
+\- One focused task per subagent — no bundling
+
+\- For complex problems, throw more compute at it via parallel subagents
+
+\- Synthesize results back into the main thread; don't chain noise
+
+
+
+\---
+
+
+
+\## 6. Autonomous Bug Fixing
+
+When given a bug report: fix it. No hand-holding required.
+
+
+
+\- Point at logs, errors, failing tests — then resolve them
+
+\- Go fix failing CI without being asked how
+
+\- Zero context-switching required from the user
+
+
+
+\---
+
+
+
+\## 7. Self-Improvement Loop
+
+After any correction, extract the pattern and prevent recurrence.
+
+
+
+\- Update `tasks/lessons.md` with: what went wrong, why, and the rule that prevents it
+
+\- Review `tasks/lessons.md` at session start for the current project
+
+\- Ruthlessly iterate until the mistake rate on that class of error drops to zero
+
+
+
+\---
+
+
+
+\## Task Execution Protocol
+
+
+
+1\. \*\*Plan first\*\* — write plan to `tasks/todo.md` with checkable items
+
+2\. \*\*Check in\*\* — verify plan before starting implementation
+
+3\. \*\*Track progress\*\* — mark items complete as you go
+
+4\. \*\*Explain changes\*\* — high-level summary at each step
+
+5\. \*\*Document results\*\* — add a review section to `tasks/todo.md`
+
+6\. \*\*Capture lessons\*\* — update `tasks/lessons.md` after any correction
+
diff --git a/find_offsets.py b/find_offsets.py
new file mode 100644
index 0000000000000000000000000000000000000000..85591b120f9c8afa8511dcfc851f7b6e639d1c4a
--- /dev/null
+++ b/find_offsets.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# find_offsets.py - Find correct character offsets for the test text
+
+text = "ذهبو الى المدرسة ثم ذهبو الى البيت ثم ذهبو مرة اخرى"
+
+print(f"Text: {text}")
+print(f"Length: {len(text)} characters\n")
+
+# Find all occurrences of "ذهبو"
+word = "ذهبو"
+print(f'Finding all occurrences of "{word}":\n')
+
+start = 0
+occurrence = 1
+offsets = []
+
+while True:
+ pos = text.find(word, start)
+ if pos == -1:
+ break
+ end = pos + len(word)
+ offsets.append((pos, end))
+ print(f"Occurrence {occurrence}: [{pos}:{end}] = '{text[pos:end]}'")
+ start = pos + 1
+ occurrence += 1
+
+print(f"\nCorrect test offsets:")
+print([{"start": start, "end": end, "original": "ذهبو", "correction": "ذهبوا", "type": "spelling"} for start, end in offsets])
diff --git a/inspect_decoder.py b/inspect_decoder.py
new file mode 100644
index 0000000000000000000000000000000000000000..5b1a6b75fa617c27933e7037efde214d4ded83cf
--- /dev/null
+++ b/inspect_decoder.py
@@ -0,0 +1,29 @@
+
+import torch
+from pathlib import Path
+
+model_path = Path(r"e:\تقليد جرامرلي\BAYAN\models\Spelling\Model\Spelling RAW.pt")
+
+try:
+ print(f"Loading {model_path}...")
+ checkpoint = torch.load(model_path, map_location="cpu")
+ state_dict = checkpoint["model_state_dict"]
+
+ print("Checking decoder keys...")
+ found_bert = False
+ for k in state_dict.keys():
+ if k.startswith("decoder.bert"):
+ print(f"Found: {k}")
+ found_bert = True
+ break
+
+ if not found_bert:
+ print("No decoder.bert keys found.")
+ # Check what IS in decoder
+ for k in state_dict.keys():
+ if k.startswith("decoder.") and not "cls" in k:
+ print(f"Decoder key: {k}")
+ break
+
+except Exception as e:
+ print(f"Error: {e}")
diff --git a/inspect_model.py b/inspect_model.py
new file mode 100644
index 0000000000000000000000000000000000000000..7b7ba7dbdfa2b53ce2f1aaa0d1190d937a8c98c7
--- /dev/null
+++ b/inspect_model.py
@@ -0,0 +1,33 @@
+
+import torch
+import sys
+from pathlib import Path
+
+model_path = Path(r"e:\تقليد جرامرلي\BAYAN\models\Spelling\Model\Spelling RAW.pt")
+
+try:
+ print(f"Loading {model_path}...")
+ # Load on CPU to be safe and fast
+ state_dict = torch.load(model_path, map_location="cpu")
+
+ if isinstance(state_dict, dict):
+ print("Loaded as dict. Keys:")
+ keys = list(state_dict.keys())
+ print(f"Total keys: {len(keys)}")
+ print("First 20 keys:")
+ for k in keys[:20]:
+ print(k)
+
+ # Check for specific architectures
+ if any(k.startswith("bert.") for k in keys):
+ print("\nLooks like BERT.")
+ elif any(k.startswith("roberta.") for k in keys):
+ print("\nLooks like RoBERTa.")
+ elif any(k.startswith("transformer.") for k in keys):
+ print("\nLooks like a Transformer.")
+ else:
+ print(f"Loaded object type: {type(state_dict)}")
+ print(state_dict)
+
+except Exception as e:
+ print(f"Error: {e}")
diff --git a/inspect_model_details.py b/inspect_model_details.py
new file mode 100644
index 0000000000000000000000000000000000000000..30625d725565028552915043c8b3ab11a90cf67c
--- /dev/null
+++ b/inspect_model_details.py
@@ -0,0 +1,53 @@
+
+import torch
+import sys
+from pathlib import Path
+
+model_path = Path(r"e:\تقليد جرامرلي\BAYAN\models\Spelling\Model\Spelling RAW.pt")
+
+try:
+ print(f"Loading {model_path}...")
+ checkpoint = torch.load(model_path, map_location="cpu")
+ state_dict = checkpoint["model_state_dict"]
+
+ # 1. Vocab and Hidden Size
+ if "encoder.embeddings.word_embeddings.weight" in state_dict:
+ shape = state_dict["encoder.embeddings.word_embeddings.weight"].shape
+ print(f"Vocab Size: {shape[0]}")
+ print(f"Hidden Size: {shape[1]}")
+ else:
+ print("Embeddings weight not found at 'encoder.embeddings.word_embeddings.weight'")
+
+ # Try finding any embedding weight
+ for k in state_dict.keys():
+ if "embeddings.word_embeddings.weight" in k:
+ print(f"Found embedding: {k} -> {state_dict[k].shape}")
+
+ # 2. Number of layers
+ layers = set()
+ for k in state_dict.keys():
+ if "encoder.encoder.layer." in k:
+ # Extract layer number
+ parts = k.split(".")
+ try:
+ # Find the part after 'layer'
+ idx = parts.index("layer")
+ layer_num = int(parts[idx+1])
+ layers.add(layer_num)
+ except:
+ pass
+
+ if layers:
+ print(f"Number of layers: {max(layers) + 1}")
+ else:
+ print("No layers found matching 'encoder.encoder.layer.X'")
+
+ # 3. Check for Heads
+ print("\nPotential Head Keys:")
+ head_keywords = ["cls", "linear", "out", "start", "end", "classifier", "generator"]
+ for k in state_dict.keys():
+ if any(kw in k for kw in head_keywords) and not "attention" in k and not "intermediate" in k and not k.startswith("encoder.encoder."):
+ print(f"{k} -> {state_dict[k].shape}")
+
+except Exception as e:
+ print(f"Error: {e}")
diff --git a/inspect_model_weights.py b/inspect_model_weights.py
new file mode 100644
index 0000000000000000000000000000000000000000..35e6b6e0ded638c08e65aeadda9dee5a58e0b89e
--- /dev/null
+++ b/inspect_model_weights.py
@@ -0,0 +1,34 @@
+
+import torch
+import sys
+from pathlib import Path
+
+model_path = Path(r"e:\تقليد جرامرلي\BAYAN\models\Spelling\Model\Spelling RAW.pt")
+
+try:
+ print(f"Loading {model_path}...")
+ checkpoint = torch.load(model_path, map_location="cpu")
+
+ if "model_state_dict" in checkpoint:
+ state_dict = checkpoint["model_state_dict"]
+ print("Found model_state_dict. Keys:")
+ keys = list(state_dict.keys())
+ print(f"Total weight keys: {len(keys)}")
+ print("First 20 weight keys:")
+ for k in keys[:20]:
+ print(k)
+
+ # Check architecture
+ if any(k.startswith("bert.") for k in keys):
+ print("\nArchitecture likely: BERT")
+ elif any(k.startswith("roberta.") for k in keys):
+ print("\nArchitecture likely: RoBERTa")
+ elif any(k.startswith("transformer.") for k in keys): # often T5
+ print("\nArchitecture likely: Transformer (T5/Bart)")
+ elif any(k.startswith("model.encoder.") for k in keys): # mBART/Bart
+ print("\nArchitecture likely: mBART/Bart")
+ else:
+ print("model_state_dict not found in checkpoint.")
+
+except Exception as e:
+ print(f"Error: {e}")
diff --git a/reproduce_issue.py b/reproduce_issue.py
new file mode 100644
index 0000000000000000000000000000000000000000..369ef7ddad5597cfae7993473eaa621198da06d5
--- /dev/null
+++ b/reproduce_issue.py
@@ -0,0 +1,60 @@
+
+import sys
+import os
+from pathlib import Path
+import logging
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Add src to path
+src_path = Path(__file__).parent / 'src'
+sys.path.insert(0, str(src_path))
+
+try:
+ from model_loader import SpellingModel, GrammarModel, PunctuationModel
+except ImportError as e:
+ logger.error(f"Failed to import models: {e}")
+ sys.exit(1)
+
+def test_models():
+ print("--- Testing Model Loading and Inference ---")
+
+ # Test Spelling
+ print("\n1. Testing Spelling Model...")
+ try:
+ spelling = SpellingModel()
+ text = "الكتبة الصحيحه مهمة"
+ corrected = spelling.correct(text)
+ print(f"Input: {text}")
+ print(f"Output: {corrected}")
+ if text == corrected:
+ print("WARNING: Spelling model returned original text (no correction).")
+ except Exception as e:
+ print(f"ERROR: Spelling model failed: {e}")
+
+ # Test Grammar
+ print("\n2. Testing Grammar Model...")
+ try:
+ grammar = GrammarModel()
+ text = "الطلاب ذهبوا الى المدرسة" # Should be gold standard? Or maybe "الطلاب ذهب" needs correction
+ corrected = grammar.correct(text)
+ print(f"Input: {text}")
+ print(f"Output: {corrected}")
+ except Exception as e:
+ print(f"ERROR: Grammar model failed: {e}")
+
+ # Test Punctuation
+ print("\n3. Testing Punctuation Model...")
+ try:
+ punctuation = PunctuationModel()
+ text = "قال المعلم الدرس مهم"
+ punctuated = punctuation.add_punctuation(text)
+ print(f"Input: {text}")
+ print(f"Output: {punctuated}")
+ except Exception as e:
+ print(f"ERROR: Punctuation model failed: {e}")
+
+if __name__ == "__main__":
+ test_models()
diff --git a/requirements.txt b/requirements.txt
index d808f36bff53e060cd5263a06338d161d6ee0ed3..e562292c9a812077953570830fd3d650fd6628bd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,3 +8,5 @@ numpy
pandas
flask
flask-cors
+jellyfish
+python-Levenshtein
diff --git a/run_app.py b/run_app.py
index ba0a9994b2886cff2b8274fb8dadf7c796e3e3a6..4df351827cddf420ea8c30759c76d9608a607e36 100644
--- a/run_app.py
+++ b/run_app.py
@@ -16,7 +16,7 @@ os.chdir(src_path)
# Import and run the app
if __name__ == '__main__':
- from app import app, load_model
+ from app import app, load_models
import logging
logging.basicConfig(
@@ -27,17 +27,18 @@ if __name__ == '__main__':
logger = logging.getLogger(__name__)
logger.info("Starting Bayan application...")
- # Load model
- if not load_model():
- logger.error("Failed to load model. Server will start but summarization will not work.")
- logger.error("Please check that the model files are in the correct location.")
+ # Load models
+ if not load_models():
+ logger.error("Failed to load any models. Server will start but functionality will be limited.")
+ logger.error("Please check that the model files are in the correct locations.")
# Run the app
port = int(os.environ.get('PORT', 5000))
- debug = os.environ.get('DEBUG', 'False').lower() == 'true'
+ debug = os.environ.get('DEBUG', 'True').lower() == 'true'
logger.info(f"Starting server on http://localhost:{port}")
logger.info("Press Ctrl+C to stop the server")
- app.run(host='0.0.0.0', port=port, debug=debug)
+ # Disable reloader to avoid loading models twice
+ app.run(host='0.0.0.0', port=port, debug=debug, use_reloader=False)
diff --git a/src/app.py b/src/app.py
index 7965c824fc4d47206e347466e47e68c4983f2c0d..9a12b0a3d868c05a566e9fbe36d17fa207d87bfd 100644
--- a/src/app.py
+++ b/src/app.py
@@ -5,12 +5,31 @@ Provides API endpoints for the Bayan web application.
import os
import logging
+import time
from flask import Flask, request, jsonify
from flask_cors import CORS
from pathlib import Path
import traceback
+import difflib
+import re
-from model_loader import SummarizationModel
+from model_loader import (
+ SummarizationModel,
+ SpellingModel,
+ AutocompleteModel,
+ GrammarModel,
+ PunctuationModel,
+ SUMMARIZATION_PATH,
+ SPELLING_PATH,
+ AUTOCOMPLETE_PATH,
+ GRAMMAR_PATH,
+ PUNCTUATION_PATH
+)
+
+HUGGINGFACE_SUMMARIZATION_REPO = os.environ.get(
+ "SUMMARIZATION_REPO_ID",
+ "bayan10/summarization-model",
+)
# Configure logging
logging.basicConfig(
@@ -28,52 +47,41 @@ MAX_TEXT_LENGTH = 5000 # Maximum characters for input text
MAX_SUMMARY_LENGTH = 512 # Maximum tokens for summary
MIN_TEXT_LENGTH = 10 # Minimum characters for summarization
-# Global model instance
-model = None
-
-
-def get_model_path():
- """Get the model path, handling different possible locations."""
- base_path = Path(__file__).parent.parent
- current_dir = Path.cwd()
-
- # Try different possible paths
- possible_paths = [
- base_path / "models" / "arabic_summarization_model" / "content" / "drive" / "MyDrive" / "arabic_summarization_model",
- base_path / "models" / "arabic_summarization_model",
- current_dir / "models" / "arabic_summarization_model" / "content" / "drive" / "MyDrive" / "arabic_summarization_model",
- current_dir / "models" / "arabic_summarization_model",
- Path("models") / "arabic_summarization_model" / "content" / "drive" / "MyDrive" / "arabic_summarization_model",
- Path("models") / "arabic_summarization_model",
- ]
-
- for path in possible_paths:
- path = path.resolve() # Resolve to absolute path
- if path.exists() and (path / "config.json").exists():
- logger.info(f"Found model at: {path}")
- return str(path)
-
- # Provide helpful error message
- error_msg = f"Model not found. Searched in:\n"
- for p in possible_paths:
- error_msg += f" - {p.resolve()}\n"
- error_msg += "\nPlease ensure the model files are in one of these locations."
- raise FileNotFoundError(error_msg)
-
-
-def load_model():
- """Load the summarization model."""
- global model
+# Global model instances
+summarization_model = None
+spelling_model = None
+autocomplete_model = None
+grammar_model = None
+punctuation_model = None
+
+
+def load_models():
+ """Load only the summarization model."""
+ global summarization_model, spelling_model, autocomplete_model, grammar_model, punctuation_model
+
+ loaded = []
+ failed = []
+
+ # Load only the Summarization model. This avoids heavy startup cost from other models.
try:
- model_path = get_model_path()
- logger.info(f"Loading model from: {model_path}")
- model = SummarizationModel(model_path)
- logger.info("Model loaded successfully")
- return True
+ logger.info(f"Loading summarization model from Hugging Face: {HUGGINGFACE_SUMMARIZATION_REPO}")
+ try:
+ summarization_model = SummarizationModel(HUGGINGFACE_SUMMARIZATION_REPO)
+ except Exception as remote_error:
+ logger.warning(f"Remote load failed, falling back to local model: {remote_error}")
+ logger.info(f"Loading summarization model from local path: {SUMMARIZATION_PATH}")
+ summarization_model = SummarizationModel(SUMMARIZATION_PATH)
+ loaded.append("summarization")
+ logger.info("Summarization model loaded successfully")
except Exception as e:
- logger.error(f"Error loading model: {str(e)}")
- logger.error(traceback.format_exc())
- return False
+ failed.append(("summarization", str(e)))
+ logger.error(f"Failed to load summarization model: {str(e)}")
+
+ logger.info(f"Models loaded: {loaded}")
+ if failed:
+ logger.warning(f"Models failed to load: {[f[0] for f in failed]}")
+
+ return len(loaded) > 0
@app.route('/')
@@ -87,7 +95,13 @@ def health_check():
"""Health check endpoint."""
return jsonify({
'status': 'healthy',
- 'model_loaded': model is not None
+ 'models': {
+ 'summarization': summarization_model is not None,
+ 'spelling': spelling_model is not None,
+ 'autocomplete': autocomplete_model is not None,
+ 'grammar': grammar_model is not None,
+ 'punctuation': punctuation_model is not None
+ }
})
@@ -103,9 +117,9 @@ def summarize():
"full_text": true/false (whether to summarize full text or just first paragraph)
}
"""
- if model is None:
+ if summarization_model is None:
return jsonify({
- 'error': 'Model not loaded. Please check server logs.',
+ 'error': 'Summarization model not loaded. Please check server logs.',
'status': 'error'
}), 503
@@ -154,7 +168,7 @@ def summarize():
# Generate summary
logger.info(f"Generating summary: length={length}, max_length={max_length}, text_length={len(text)}")
- summary = model.summarize(text, max_length=max_length, min_length=max(10, max_length // 3))
+ summary = summarization_model.summarize(text, max_length=max_length, min_length=max(10, max_length // 3))
return jsonify({
'summary': summary,
@@ -180,6 +194,489 @@ def summarize():
}), 500
+@app.route('/api/spelling', methods=['POST'])
+def spelling_correction():
+ """
+ Correct spelling in Arabic text.
+
+ Expected JSON payload:
+ {
+ "text": "Arabic text to correct"
+ }
+ """
+ if spelling_model is None:
+ return jsonify({
+ 'error': 'Spelling model not loaded. Please check server logs.',
+ 'status': 'error'
+ }), 503
+
+ try:
+ if not request.is_json:
+ return jsonify({'error': 'Request must be JSON', 'status': 'error'}), 400
+
+ data = request.get_json()
+ text = data.get('text', '').strip()
+
+ if not text:
+ return jsonify({'error': 'Text is required', 'status': 'error'}), 400
+
+ logger.info(f"Correcting spelling for text of length: {len(text)}")
+ corrected = spelling_model.correct(text)
+
+ return jsonify({
+ 'corrected': corrected,
+ 'status': 'success',
+ 'original_length': len(text),
+ 'corrected_length': len(corrected)
+ })
+
+ except Exception as e:
+ logger.error(f"Error during spelling correction: {str(e)}")
+ logger.error(traceback.format_exc())
+ return jsonify({
+ 'error': 'An error occurred during spelling correction.',
+ 'status': 'error',
+ 'details': str(e) if app.debug else None
+ }), 500
+
+
+@app.route('/api/autocomplete', methods=['POST'])
+def autocomplete():
+ """
+ Get autocomplete suggestions for Arabic text.
+
+ Expected JSON payload:
+ {
+ "text": "Arabic text prefix",
+ "n": 5 (number of suggestions, optional)
+ }
+ """
+ if autocomplete_model is None:
+ return jsonify({
+ 'error': 'Autocomplete model not loaded. Please check server logs.',
+ 'status': 'error'
+ }), 503
+
+ try:
+ if not request.is_json:
+ return jsonify({'error': 'Request must be JSON', 'status': 'error'}), 400
+
+ data = request.get_json()
+ text = data.get('text', '').strip()
+ n = int(data.get('n', 5))
+
+ if not text:
+ return jsonify({'error': 'Text is required', 'status': 'error'}), 400
+
+ logger.info(f"Getting autocomplete suggestions for: {text[:50]}...")
+ suggestions = autocomplete_model.predict(text, n=n)
+ logger.info(f"Autocomplete suggestions (n={n}): {suggestions}")
+
+ return jsonify({
+ 'suggestions': suggestions,
+ 'status': 'success'
+ })
+
+ except Exception as e:
+ logger.error(f"Error during autocomplete: {str(e)}")
+ logger.error(traceback.format_exc())
+ return jsonify({
+ 'error': 'An error occurred during autocomplete.',
+ 'status': 'error',
+ 'details': str(e) if app.debug else None
+ }), 500
+
+
+@app.route('/api/grammar', methods=['POST'])
+def grammar_correction():
+ """
+ Correct grammar in Arabic text.
+
+ Expected JSON payload:
+ {
+ "text": "Arabic text to correct"
+ }
+ """
+ if grammar_model is None:
+ return jsonify({
+ 'error': 'Grammar model not loaded. Please check server logs.',
+ 'status': 'error'
+ }), 503
+
+ try:
+ if not request.is_json:
+ return jsonify({'error': 'Request must be JSON', 'status': 'error'}), 400
+
+ data = request.get_json()
+ text = data.get('text', '').strip()
+
+ if not text:
+ return jsonify({'error': 'Text is required', 'status': 'error'}), 400
+
+ logger.info(f"Correcting grammar for text of length: {len(text)}")
+ corrected = grammar_model.correct(text)
+
+ return jsonify({
+ 'corrected': corrected,
+ 'status': 'success',
+ 'original_length': len(text),
+ 'corrected_length': len(corrected)
+ })
+
+ except Exception as e:
+ logger.error(f"Error during grammar correction: {str(e)}")
+ logger.error(traceback.format_exc())
+ return jsonify({
+ 'error': 'An error occurred during grammar correction.',
+ 'status': 'error',
+ 'details': str(e) if app.debug else None
+ }), 500
+
+
+@app.route('/api/punctuation', methods=['POST'])
+def add_punctuation():
+ """
+ Add punctuation to Arabic text.
+
+ Expected JSON payload:
+ {
+ "text": "Arabic text without punctuation"
+ }
+ """
+ if punctuation_model is None:
+ return jsonify({
+ 'error': 'Punctuation model not loaded. Please check server logs.',
+ 'status': 'error'
+ }), 503
+
+ try:
+ if not request.is_json:
+ return jsonify({'error': 'Request must be JSON', 'status': 'error'}), 400
+
+ data = request.get_json()
+ text = data.get('text', '').strip()
+
+ if not text:
+ return jsonify({'error': 'Text is required', 'status': 'error'}), 400
+
+ logger.info(f"Adding punctuation for text of length: {len(text)}")
+ punctuated = punctuation_model.add_punctuation(text)
+
+ return jsonify({
+ 'punctuated': punctuated,
+ 'status': 'success',
+ 'original_length': len(text),
+ 'punctuated_length': len(punctuated)
+ })
+
+ except Exception as e:
+ logger.error(f"Error during punctuation: {str(e)}")
+ logger.error(traceback.format_exc())
+ return jsonify({
+ 'error': 'An error occurred during punctuation.',
+ 'status': 'error',
+ 'details': str(e) if app.debug else None
+ }), 500
+
+
+def get_word_positions(text):
+ """
+ Returns a list of tuples (word, start_char_index, end_char_index)
+ for all whitespace-separated words in the text.
+ """
+ positions = []
+ for m in re.finditer(r'\S+', text):
+ positions.append((m.group(), m.start(), m.end()))
+ return positions
+
+
+class OffsetMapper:
+ def __init__(self, original, modified):
+ self.original = original
+ self.modified = modified
+ self.mapping = [] # list of (mod_start, mod_end, orig_start, orig_end)
+ self._build_mapping()
+
+ def _build_mapping(self):
+ s = difflib.SequenceMatcher(None, self.original, self.modified)
+ for tag, i1, i2, j1, j2 in s.get_opcodes():
+ self.mapping.append((j1, j2, i1, i2))
+
+ def map_offset(self, mod_offset):
+ """
+ Given a character offset in the modified text, return the corresponding
+ character offset in the original text.
+ """
+ for j1, j2, i1, i2 in self.mapping:
+ if j1 <= mod_offset <= j2:
+ if j2 == j1: # insertion point
+ return i1
+ # Proportional mapping inside the block
+ ratio = (mod_offset - j1) / (j2 - j1)
+ return int(i1 + ratio * (i2 - i1))
+ return len(self.original)
+
+
+def get_word_diffs(original, corrected):
+ """
+ Identify differences between original and corrected text at the word level.
+ Returns a list of suggestions with start and end character offsets.
+ """
+ orig_words = get_word_positions(original)
+ corr_words = get_word_positions(corrected)
+ s = difflib.SequenceMatcher(None, [w[0] for w in orig_words], [w[0] for w in corr_words])
+ suggestions = []
+
+ for tag, i1, i2, j1, j2 in s.get_opcodes():
+ if tag == 'replace':
+ if i1 < len(orig_words) and i2 - 1 < len(orig_words):
+ start_char = orig_words[i1][1]
+ end_char = orig_words[i2-1][2]
+ suggestions.append({
+ 'start': start_char,
+ 'end': end_char,
+ 'original': original[start_char:end_char],
+ 'correction': " ".join([w[0] for w in corr_words[j1:j2]]),
+ 'type': 'generic'
+ })
+ elif tag == 'delete':
+ if i1 < len(orig_words) and i2 - 1 < len(orig_words):
+ start_char = orig_words[i1][1]
+ end_char = orig_words[i2-1][2]
+ suggestions.append({
+ 'start': start_char,
+ 'end': end_char,
+ 'original': original[start_char:end_char],
+ 'correction': '',
+ 'type': 'generic'
+ })
+ elif tag == 'insert':
+ pos = orig_words[i1][1] if i1 < len(orig_words) else len(original)
+ suggestions.append({
+ 'start': pos,
+ 'end': pos,
+ 'original': '',
+ 'correction': " ".join([w[0] for w in corr_words[j1:j2]]),
+ 'type': 'generic'
+ })
+
+ return suggestions
+
+
+def _levenshtein(a, b):
+ """Simple Levenshtein distance for short words."""
+ m, n = len(a), len(b)
+ if m == 0:
+ return n
+ if n == 0:
+ return m
+ dp = [[0] * (n + 1) for _ in range(m + 1)]
+ for i in range(m + 1):
+ dp[i][0] = i
+ for j in range(n + 1):
+ dp[0][j] = j
+ for i in range(1, m + 1):
+ for j in range(1, n + 1):
+ cost = 0 if a[i - 1] == b[j - 1] else 1
+ dp[i][j] = min(
+ dp[i - 1][j] + 1, # deletion
+ dp[i][j - 1] + 1, # insertion
+ dp[i - 1][j - 1] + cost, # substitution
+ )
+ return dp[m][n]
+
+
+def _is_small_spelling_change(orig_word, corr_word):
+ """
+ Heuristic: only accept small spelling edits and ignore
+ aggressive changes (to avoid over-editing).
+ """
+ if not orig_word or not corr_word:
+ return False
+ if orig_word == corr_word:
+ return False
+
+ # Ignore tokens that contain non-letters (numbers / punctuation)
+ # Arabic letters range plus basic Latin letters.
+ if re.search(r'[^ء-يآأإىa-zA-Z]', orig_word):
+ return False
+ if re.search(r'[^ء-يآأإىa-zA-Z]', corr_word):
+ return False
+
+ dist = _levenshtein(orig_word, corr_word)
+ max_len = max(len(orig_word), len(corr_word))
+
+ # Allow at most 2 character edits and at most 40% of the word
+ return dist <= 2 and (dist / max_len) <= 0.4
+
+
+@app.route('/api/analyze', methods=['POST'])
+def analyze_text():
+ """
+ Perform sequential analysis (Spelling -> Grammar -> Punctuation)
+ and return word-level suggestions with offsets.
+ """
+ try:
+ if not request.is_json:
+ return jsonify({'error': 'Request must be JSON', 'status': 'error'}), 400
+
+ data = request.get_json()
+ text = data.get('text', '').strip()
+
+ if not text:
+ return jsonify({'error': 'Text is required', 'status': 'error'}), 400
+
+ current_text = text
+ suggestions = []
+ mappers = []
+ total_start = time.time()
+
+ def map_range_to_original(start, end):
+ curr_start, curr_end = start, end
+ for mapper in reversed(mappers):
+ curr_start = mapper.map_offset(curr_start)
+ curr_end = mapper.map_offset(curr_end)
+ return curr_start, curr_end
+
+ # 1. Spelling (with conservative post-filtering to avoid over-editing)
+ if spelling_model:
+ try:
+ t0 = time.time()
+ logger.info(f"[ANALYZE] Step 1: Spelling correction starting...")
+ raw_corrected = spelling_model.correct(current_text)
+ logger.info(f"[ANALYZE] Step 1: Spelling done in {time.time()-t0:.2f}s")
+
+ if raw_corrected != current_text:
+ orig_word_positions = get_word_positions(current_text)
+ corr_word_positions = get_word_positions(raw_corrected)
+
+ orig_word_strings = [w[0] for w in orig_word_positions]
+ corr_word_strings = [w[0] for w in corr_word_positions]
+
+ s = difflib.SequenceMatcher(None, orig_word_strings, corr_word_strings)
+ new_words = []
+
+ for tag, i1, i2, j1, j2 in s.get_opcodes():
+ if tag == 'equal':
+ start_idx = orig_word_positions[i1][1]
+ end_idx = orig_word_positions[i2-1][2]
+ new_words.append(current_text[start_idx:end_idx])
+ elif tag == 'replace':
+ o_segment = orig_word_strings[i1:i2]
+ c_segment = corr_word_strings[j1:j2]
+
+ start_idx = orig_word_positions[i1][1]
+ end_idx = orig_word_positions[i2-1][2]
+
+ if len(o_segment) == 1 and len(c_segment) == 1:
+ # 1-word → 1-word: accept only small edits (typos)
+ o_word = o_segment[0]
+ c_word = c_segment[0]
+ if _is_small_spelling_change(o_word, c_word):
+ new_words.append(c_word)
+ suggestions.append({
+ 'start': start_idx,
+ 'end': end_idx,
+ 'original': o_word,
+ 'correction': c_word,
+ 'type': 'spelling',
+ })
+ else:
+ new_words.append(current_text[start_idx:end_idx])
+ elif len(o_segment) == 1 and len(c_segment) > 1:
+ # 1-word → N words: accept when original is long (likely concatenated)
+ o_word = o_segment[0]
+ if len(o_word) >= 12 and ' ' not in o_word:
+ corr_str = " ".join(c_segment)
+ new_words.append(corr_str)
+ suggestions.append({
+ 'start': start_idx,
+ 'end': end_idx,
+ 'original': o_word,
+ 'correction': corr_str,
+ 'type': 'spelling',
+ })
+ else:
+ new_words.append(current_text[start_idx:end_idx])
+ else:
+ new_words.extend([current_text[orig_word_positions[idx][1]:orig_word_positions[idx][2]] for idx in range(i1, i2)])
+ elif tag == 'delete':
+ for idx in range(i1, i2):
+ new_words.append(current_text[orig_word_positions[idx][1]:orig_word_positions[idx][2]])
+ elif tag == 'insert':
+ continue
+
+ safe_text = " ".join(new_words)
+ mappers.append(OffsetMapper(current_text, safe_text))
+ current_text = safe_text
+ except Exception as e:
+ logger.error(f"[ANALYZE] Spelling failed: {e}")
+
+ # 2. Grammar (runs on spelling-corrected text)
+ if grammar_model:
+ try:
+ t0 = time.time()
+ logger.info(f"[ANALYZE] Step 2: Grammar correction starting...")
+ corrected_grammar = grammar_model.correct(current_text)
+ logger.info(f"[ANALYZE] Step 2: Grammar done in {time.time()-t0:.2f}s")
+ if corrected_grammar != current_text:
+ diffs = get_word_diffs(current_text, corrected_grammar)
+ for d in diffs:
+ orig_start, orig_end = map_range_to_original(d['start'], d['end'])
+ suggestions.append({
+ 'start': orig_start,
+ 'end': orig_end,
+ 'original': text[orig_start:orig_end],
+ 'correction': d['correction'],
+ 'type': 'grammar'
+ })
+ mappers.append(OffsetMapper(current_text, corrected_grammar))
+ current_text = corrected_grammar
+ except Exception as e:
+ logger.error(f"[ANALYZE] Grammar failed: {e}")
+
+ # 3. Punctuation (runs on grammar-corrected text)
+ if punctuation_model:
+ try:
+ t0 = time.time()
+ logger.info(f"[ANALYZE] Step 3: Punctuation starting...")
+ corrected_punc = punctuation_model.add_punctuation(current_text)
+ logger.info(f"[ANALYZE] Step 3: Punctuation done in {time.time()-t0:.2f}s")
+ if corrected_punc != current_text:
+ diffs = get_word_diffs(current_text, corrected_punc)
+ for d in diffs:
+ orig_start, orig_end = map_range_to_original(d['start'], d['end'])
+ suggestions.append({
+ 'start': orig_start,
+ 'end': orig_end,
+ 'original': text[orig_start:orig_end],
+ 'correction': d['correction'],
+ 'type': 'punctuation'
+ })
+ current_text = corrected_punc
+ except Exception as e:
+ logger.error(f"[ANALYZE] Punctuation failed: {e}")
+
+ total_time = time.time() - total_start
+ logger.info(f"[ANALYZE] Total analysis time: {total_time:.2f}s | Suggestions: {len(suggestions)}")
+
+ return jsonify({
+ 'original': text,
+ 'corrected': current_text,
+ 'suggestions': suggestions,
+ 'status': 'success'
+ })
+
+ except Exception as e:
+ logger.error(f"Error during analysis: {str(e)}")
+ logger.error(traceback.format_exc())
+ return jsonify({
+ 'error': 'An error occurred during text analysis.',
+ 'status': 'error',
+ 'details': str(e) if app.debug else None
+ }), 500
+
+
@app.errorhandler(404)
def not_found(error):
"""Handle 404 errors."""
@@ -200,12 +697,12 @@ def internal_error(error):
if __name__ == '__main__':
- # Load model on startup
+ # Load models on startup
logger.info("Starting Bayan server...")
- if not load_model():
- logger.error("Failed to load model. Server will start but summarization will not work.")
- logger.error("Please check that the model files are in the correct location.")
+ if not load_models():
+ logger.error("Failed to load any models. Server will start but functionality will be limited.")
+ logger.error("Please check that the model files are in the correct locations.")
# Run the app
port = int(os.environ.get('PORT', 5000))
diff --git a/src/ara_spell.py b/src/ara_spell.py
new file mode 100644
index 0000000000000000000000000000000000000000..d506fbc5d666cfdfc64dbd07b96782bbd60435d7
--- /dev/null
+++ b/src/ara_spell.py
@@ -0,0 +1,2126 @@
+# ## 📦 Part 1: Imports & Setup
+
+
+import re
+import torch
+import os
+from transformers import AutoTokenizer, EncoderDecoderModel
+import Levenshtein
+import jellyfish # NEW: For Damerau-Levenshtein (transpositions as 1 edit)
+
+print("✅ All imports successful")
+print(f"🔧 PyTorch version: {torch.__version__}")
+print(f"🔧 CUDA available: {torch.cuda.is_available()}")
+if torch.cuda.is_available():
+ print(f"🔧 GPU: {torch.cuda.get_device_name(0)}")
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# 🏆 PRODUCTION SPELL CHECKER - OPTIMIZED VERSION (Merged from src/)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# 🏆 PRODUCTION SPELL CHECKER - OPTIMIZED VERSION (Merged from src/)
+# ═══════════════════════════════════════════════════════════════════════════════
+# IMPROVEMENTS:
+# 1. unified_collapse_repeated() - More conservative (3+ for Arabic)
+# 2. fix_hamza_conservative() - Only fixes word endings
+# 3. remove_hallucinations() - Removes duplicate words and trailing 'و'
+# 4. Expanded PREPOSITIONS - 16 prepositions instead of 2
+# 5. Better word splitting and joining
+# ═══════════════════════════════════════════════════════════════════════════════
+
+from enum import Enum
+from typing import List, Tuple, Optional
+
+# ─────────────────────────────────────────────────────────────────────────────
+# ERROR TYPE ENUM
+# ─────────────────────────────────────────────────────────────────────────────
+
+class ErrorType(Enum):
+ """Types of spelling errors"""
+ CHAR_REPETITION = "char_repetition"
+ WORD_MERGE = "word_merge"
+ CHAR_SUBSTITUTION = "char_substitution"
+ MIXED = "mixed"
+ CLEAN = "clean"
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# POST PROCESSOR (OPTIMIZED - Merged from src/)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class AraSpellPostProcessor:
+ """Post-processing techniques (OPTIMIZED VERSION)"""
+
+ ARABIC_HARAKAT = 'ًٌٍَُِّْ'
+ TATWEEL = 'ـ'
+ NORMALIZER_MAP = {
+ 'ﻹ': 'لإ', 'ﻷ': 'لأ', 'ﻵ': 'لآ', 'ﻻ': 'لا', 'ﷲ': 'الله'
+ }
+ ARABIC_CONSONANTS = set('بتثجحخدذرزسشصضطظعغفقكلمن')
+
+ # ═══════════════════════════════════════════════════════════════════════════════
+ # BASIC NORMALIZATION
+ # ═══════════════════════════════════════════════════════════════════════════════
+
+ @staticmethod
+ def remove_harakat(text: str) -> str:
+ """Remove Arabic diacritics"""
+ return re.sub(r'[ً-ْ]', '', text)
+
+ @staticmethod
+ def remove_tatweel(text: str) -> str:
+ """Remove Arabic kashida/tatweel"""
+ return text.replace(AraSpellPostProcessor.TATWEEL, '')
+
+ @staticmethod
+ def normalize_special_chars(text: str) -> str:
+ """Normalize special Arabic ligatures"""
+ for old, new in AraSpellPostProcessor.NORMALIZER_MAP.items():
+ text = text.replace(old, new)
+ return text
+
+ # ═══════════════════════════════════════════════════════════════════════════════
+ # UNIFIED CORE FUNCTIONS (NEW from src/)
+ # ═══════════════════════════════════════════════════════════════════════════════
+
+ @staticmethod
+ def unified_collapse_repeated(text: str) -> str:
+ """
+ UNIFIED repetition collapse (IMPROVED!)
+ - Arabic: 3+ consecutive → 1 (more conservative!)
+ - Latin: 2+ consecutive → 1
+ """
+ # Arabic characters: 3+ → 1
+ text = re.sub(r"([\u0600-\u06FF])\1{2,}", r"\1", text)
+
+ # Latin characters: 2+ → 1
+ text = re.sub(r"([a-zA-Z])\1+", r"\1", text)
+
+ return text
+
+
+
+ @staticmethod
+ def remove_duplicate_words(text: str) -> str:
+ """
+ Remove consecutive duplicate words (NEW from src/)
+ Examples: كتاب كتاب → كتاب
+ """
+ words = text.split()
+ if len(words) < 2:
+ return text
+
+ result = [words[0]]
+ for i in range(1, len(words)):
+ if words[i] != words[i-1]:
+ result.append(words[i])
+
+ return ' '.join(result)
+
+
+
+ @staticmethod
+ def normalize_spaces(text: str) -> str:
+ """
+ UNIFIED space normalization (NEW from src/)
+ """
+ # Multiple spaces → single
+ text = re.sub(r' +', ' ', text)
+
+ # Unicode spaces
+ text = text.replace('\u00A0', ' ') # Non-breaking space
+ text = text.replace('\u200B', '') # Zero-width space
+ text = text.replace('\u200C', '') # Zero-width non-joiner
+ text = text.replace('\u200D', '') # Zero-width joiner
+
+ # Trim
+ text = text.strip()
+
+ # Punctuation spacing
+ text = re.sub(r'\s*([،؛؟!.])\s*', r'\1 ', text)
+ text = text.strip()
+
+ return text
+
+ @staticmethod
+ def remove_word_repetition_with_wa(text: str) -> str:
+ """Remove word و word → word"""
+ words = text.split()
+ result = []
+ i = 0
+ while i < len(words):
+ if i + 2 < len(words) and words[i] == words[i+2] and words[i+1] == 'و':
+ result.append(words[i])
+ i += 3
+ else:
+ result.append(words[i])
+ i += 1
+ return ' '.join(result)
+
+ # ═══════════════════════════════════════════════════════════════════════════════
+ # HAMZA HANDLING (NEW from src/)
+ # ═══════════════════════════════════════════════════════════════════════════════
+
+ @staticmethod
+ def fix_hamza_conservative(text: str) -> str:
+ """
+ CONSERVATIVE Hamza normalization (NEW!)
+ Only normalizes at word END, not middle
+
+ Examples:
+ ✓ "المدرسه" → "المدرسة" (end of word)
+ ✓ "سأل" → "سأل" (middle - KEEP IT!)
+ """
+ words = text.split()
+ result = []
+
+ for word in words:
+ if len(word) >= 3:
+ # Fix trailing أ → ا
+ if word.endswith('أ'):
+ word = word[:-1] + 'ا'
+
+ # Fix trailing إ → ا
+ if word.endswith('إ'):
+ word = word[:-1] + 'ا'
+
+ result.append(word)
+
+ return ' '.join(result)
+
+ @staticmethod
+ def fix_ha_ta_marbuta(text: str) -> str:
+ """
+ Fix ه → ة at end of words (pattern-based)
+ """
+ words = text.split()
+ result = []
+
+ for word in words:
+ if len(word) >= 4 and word.endswith('ه'):
+ # Check if second-to-last char is a consonant
+ if word[-2] in AraSpellPostProcessor.ARABIC_CONSONANTS:
+ result.append(word[:-1] + 'ة')
+ continue
+ result.append(word)
+
+ return ' '.join(result)
+
+ # ═══════════════════════════════════════════════════════════════════════════════
+ # HALLUCINATION REMOVAL (NEW from src/)
+ # ═══════════════════════════════════════════════════════════════════════════════
+
+ @staticmethod
+ def remove_hallucinations(text: str) -> str:
+ """
+ Remove model hallucinations (NEW from src/):
+ - Duplicate words
+ - Trailing 'و' artifacts
+ """
+ words = text.split()
+ if not words:
+ return text
+
+ result = []
+ i = 0
+
+ def normalize_word(w: str) -> str:
+ """Normalize for comparison"""
+ w = w.replace('ال', '').replace('ة', 'ه')
+ w = re.sub(r'[أإآ]', 'ا', w)
+ return w
+
+ while i < len(words):
+ word = words[i]
+
+ # Remove trailing 'و' artifacts (الماضيةو → الماضية)
+ if len(word) > 4 and word.endswith('و'):
+ prev_char = word[-2]
+ if prev_char in 'ةهاأإآء':
+ word = word[:-1]
+
+ # Check for duplicate patterns
+ if i + 1 < len(words):
+ next_word = words[i + 1]
+ if normalize_word(word) == normalize_word(next_word):
+ # Keep the one with 'ال' if possible
+ keep = next_word if next_word.startswith('ال') and not word.startswith('ال') else word
+ result.append(keep)
+ i += 2
+ continue
+
+ result.append(word)
+ i += 1
+
+ return ' '.join(result)
+
+ @staticmethod
+ def remove_hallucinated_prefix(text: str, original: str) -> str:
+ """Remove particles (و/في) added by model if not in original"""
+ if not original:
+ return text
+
+ if text.startswith('و ') and not original.startswith('و'):
+ rest = text[2:].strip()
+ # Verify it matches original
+ if AraSpellPostProcessor.normalize_special_chars(rest) == AraSpellPostProcessor.normalize_special_chars(original):
+ return rest
+
+ return text
+
+ # ═══════════════════════════════════════════════════════════════════════════════
+ # WORD SPLITTING & MERGING (NEW from src/)
+ # ═══════════════════════════════════════════════════════════════════════════════
+
+ @staticmethod
+ def merge_separated_al(text: str) -> str:
+ """Merge 'ال' separated by space: ال + كتاب → الكتاب"""
+ return re.sub(r'\bال\s+(\w+)', r'ال\1', text)
+
+ @staticmethod
+ def join_fragments(text: str) -> str:
+ """
+ IMPROVED: Join short fragments with better validation (NEW from src/)
+ الط + الب → الطالب
+
+ FIXED: No longer joins valid separate words like "في من"
+ """
+ words = text.split()
+ if len(words) < 2:
+ return text
+
+ # Common standalone words that should NOT be merged
+ STANDALONE_WORDS = {
+ 'من', 'في', 'على', 'عن', 'مع', 'إلى', 'الى', 'حتى', 'منذ', 'خلال',
+ 'بعد', 'قبل', 'ب', 'ل', 'ك', 'و', 'أو', 'لا', 'ما', 'لم', 'لن',
+ 'هو', 'هي', 'هم', 'أن', 'إن', 'كل', 'كان', 'قد', 'قال', 'ذلك',
+ 'هذا', 'هذه', 'تلك', 'التي', 'الذي', 'التى', 'اللذي'
+ }
+
+ result = []
+ i = 0
+
+ while i < len(words):
+ word = words[i]
+
+ if i + 1 < len(words):
+ next_word = words[i + 1]
+
+ # SAFETY: Don't merge if both are standalone words
+ if word in STANDALONE_WORDS and next_word in STANDALONE_WORDS:
+ result.append(word)
+ i += 1
+ continue
+
+ # Case 1: Single char fragment (safe to merge)
+ if len(next_word) == 1:
+ result.append(word + next_word)
+ i += 2
+ continue
+
+ # Case 2: Overlap (last char of word == first char of next)
+ if len(word) >= 2 and len(next_word) >= 2 and word[-1] == next_word[0]:
+ if not (word in STANDALONE_WORDS and next_word in STANDALONE_WORDS):
+ result.append(word[:-1] + next_word)
+ i += 2
+ continue
+
+ # Case 3: Short fragments (2-4 chars + 1-2 chars)
+ if (2 <= len(word) <= 4 and
+ 1 <= len(next_word) <= 2 and
+ 3 <= len(word) + len(next_word) <= 7):
+ if not (word in STANDALONE_WORDS and next_word in STANDALONE_WORDS):
+ result.append(word + next_word)
+ i += 2
+ continue
+
+ result.append(word)
+ i += 1
+
+ return ' '.join(result)
+
+ # ═══════════════════════════════════════════════════════════════════════════════
+ # MAIN PIPELINES
+ # ═══════════════════════════════════════════════════════════════════════════════
+
+ @staticmethod
+ def full_postprocess(text: str, original: str = "") -> str:
+ """
+ Apply all post-processing steps (OPTIMIZED ORDER!)
+ """
+ # 1. Remove hallucinated prefixes
+ if original:
+ text = AraSpellPostProcessor.remove_hallucinated_prefix(text, original)
+
+ # 2. Basic normalization
+ text = AraSpellPostProcessor.normalize_special_chars(text)
+
+ # 3. Remove hallucinations
+ text = AraSpellPostProcessor.remove_hallucinations(text)
+
+ # 4. Collapse repetitions (UNIFIED!)
+ text = AraSpellPostProcessor.unified_collapse_repeated(text)
+
+ # 5. Fix Hamza (CONSERVATIVE!)
+ text = AraSpellPostProcessor.fix_hamza_conservative(text)
+
+ # 6. Fix Ta Marbuta
+ text = AraSpellPostProcessor.fix_ha_ta_marbuta(text)
+
+ # 7. Remove word repetition with 'و'
+ text = AraSpellPostProcessor.remove_word_repetition_with_wa(text)
+
+ # 8. Remove duplicate words
+ text = AraSpellPostProcessor.remove_duplicate_words(text)
+
+ # 9. Final space normalization
+ text = AraSpellPostProcessor.normalize_spaces(text)
+
+ return text
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# ERROR CLASSIFIER (Keep from original notebook)
+# ─────────────────────────────────────────────────────────────────────────────
+
+class ErrorClassifier:
+ """Classify type of spelling error"""
+
+ NON_ARABIC_KEYBOARD = set('پگچژکەڕڤڵڎےۀۃھیټډڼڑ')
+
+ @staticmethod
+ def has_char_substitution(text: str) -> bool:
+ return any(c in ErrorClassifier.NON_ARABIC_KEYBOARD for c in text)
+
+ @staticmethod
+ def has_char_repetition(text: str, threshold: int = 3) -> bool:
+ return bool(re.search(r"(.)\1{" + str(threshold - 1) + ",}", text))
+
+ @staticmethod
+ def has_word_merge(text: str, max_word_len: int = 8) -> bool:
+ words = text.split()
+ if any(len(w) > max_word_len for w in words):
+ return True
+ if len(words) == 1 and len(text) > 6:
+ return True
+ return False
+
+ @staticmethod
+ def classify(text: str) -> ErrorType:
+ """Classify the error type"""
+ has_rep = ErrorClassifier.has_char_repetition(text)
+ has_merge = ErrorClassifier.has_word_merge(text)
+ has_sub = ErrorClassifier.has_char_substitution(text)
+
+ error_count = sum([has_rep, has_merge, has_sub])
+
+ if error_count >= 2:
+ return ErrorType.MIXED
+ elif has_sub:
+ return ErrorType.CHAR_SUBSTITUTION
+ elif has_rep:
+ return ErrorType.CHAR_REPETITION
+ elif has_merge:
+ return ErrorType.WORD_MERGE
+ else:
+ return ErrorType.CLEAN
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# Part 3: Production Spell Checker Class
+# This is the best pipeline based on extensive testing of 8+ different approaches
+# ═══════════════════════════════════════════════════════════════════════════════
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# RULES-BASED CORRECTOR (EXPANDED - Merged from src/)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class RulesBasedCorrector:
+ """Rules-based correction (EXPANDED VERSION)"""
+
+ # Persian/Urdu → Arabic mapping
+ SUBSTITUTION_MAP = {
+ 'ک': 'ك', 'ی': 'ي', 'ے': 'ي',
+ 'پ': 'ب', 'چ': 'ج', 'ژ': 'ز',
+ 'گ': 'ك', 'ڤ': 'ف', 'ڵ': 'ل',
+ 'ڕ': 'ر', 'ڎ': 'د', 'ڼ': 'ن',
+ 'ټ': 'ت', 'ډ': 'د', 'ړ': 'ر',
+ 'ۀ': 'ه', 'ۃ': 'ة', 'ھ': 'ه',
+ 'ە': 'ه', 'ڑ': 'ر'
+ }
+
+ # EXPANDED: 16 prepositions instead of 2
+ PREPOSITIONS = {
+ 'من', 'في', 'على', 'عن', 'مع', 'إلى', 'الى',
+ 'حتى', 'منذ', 'خلال', 'بعد', 'قبل',
+ 'ب', 'ل', 'ك',
+ 'لل'
+ }
+
+ @staticmethod
+ def fix_char_substitution(text: str) -> str:
+ """Replace Persian/Urdu characters with Arabic"""
+ for old, new in RulesBasedCorrector.SUBSTITUTION_MAP.items():
+ text = text.replace(old, new)
+ return text
+
+ @staticmethod
+ def fix_char_repetition(text: str) -> str:
+ """
+ Remove excessive character repetition (IMPROVED!)
+ Now: 3+ consecutive → 1 (more conservative)
+ """
+ # Only collapse 3+ repetitions (not 2+)
+ text = re.sub(r'([^\d\s])\1{2,}', r'\1', text)
+ return text
+
+ @staticmethod
+ def advanced_heuristic_repair(text: str) -> str:
+ """
+ Apply aggressive heuristic repairs to generate a strong baseline candidate.
+ 1. Unified Char Fixes (Persian/Urdu + Repetition)
+ 2. Aggressive Word Splitting (Iterative & Anchored)
+ """
+ # 1. Base Fixes
+ text = RulesBasedCorrector.fix_char_substitution(text)
+ text = RulesBasedCorrector.fix_char_repetition(text)
+
+ # 2. Heuristic Split
+ words = text.split()
+ processed_words = []
+ for word in words:
+ processed_words.append(RulesBasedCorrector._recursive_split(word))
+
+ return ' '.join(processed_words)
+
+ @staticmethod
+ def _recursive_split(word: str) -> str:
+ """
+ Recursively split merged words (Anchored to Start)
+ Avoids splitting 'المنزل' -> 'ال من زل' (middle split)
+ """
+ if len(word) < 4:
+ return word
+
+ # 1. Separable Prepositions (Must be at START)
+ # "فيالبيت" -> "في البيت"
+ separables = sorted(['من', 'في', 'على', 'عن', 'مع', 'إلى', 'الى', 'حتى', 'منذ', 'خلال', 'بعد', 'قبل'], key=len, reverse=True)
+
+ for sep in separables:
+ # Check matches: exact match or prefix match
+ if word == sep:
+ return word
+
+ if word.startswith(sep):
+ remainder = word[len(sep):]
+ # Condition: Remainder must be substantial (usually starts with al- or len > 2)
+ if len(remainder) >= 3:
+ # Recursive call on remainder
+ return sep + " " + RulesBasedCorrector._recursive_split(remainder)
+
+ # 2. Common typo merges (e.g. "يا" + Name)
+ if word.startswith('يا') and len(word) > 4:
+ return 'يا ' + RulesBasedCorrector._recursive_split(word[2:])
+
+ # 3. Attached Particles (Only 'Wa' and 'Fa' are commonly mistakenly merged with non-al words in typos)
+ # "وال" -> "و ال" is usually correct in tokenization but "و" is attached in script.
+ # We only split if it looks like a HARD merge error.
+
+ return word
+
+
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# OUTPUT VALIDATOR (Hallucination Prevention)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class OutputValidator:
+ """Validate model outputs to prevent hallucinations"""
+
+ @staticmethod
+ def calculate_edit_distance(s1: str, s2: str) -> int:
+ """Calculate Levenshtein distance"""
+ return Levenshtein.distance(s1, s2)
+
+ @staticmethod
+ def check_character_preservation(original: str, corrected: str) -> Tuple[bool, str]:
+ """Check if characters are mostly preserved (Jaccard similarity)"""
+ chars_original = set(original)
+ chars_corrected = set(corrected)
+
+ if not chars_original:
+ return True, "valid"
+
+ intersection = chars_original & chars_corrected
+ union = chars_original | chars_corrected
+
+ jaccard = len(intersection) / len(union) if union else 0
+
+ if jaccard < 0.35:
+ return False, "low_character_similarity"
+
+ return True, "valid"
+
+ @staticmethod
+ def check_word_count(original: str, corrected: str) -> Tuple[bool, str]:
+ """
+ Check if word count is reasonable
+ Relaxed: Allow splitting merged words (count can double)
+ """
+ len_orig = len(original.split())
+ len_corr = len(corrected.split())
+
+ # Allow expanding 1 word to up to 3 (e.g. "فيالمدرسة" -> "في المدرسة")
+ if len_orig == 1:
+ if len_corr <= 3:
+ return True, "valid"
+ # If original is very long, allow more splits (e.g. "هذاالولدذهبإلىالمدرسة")
+ if len(original) > 12 and len_corr <= 6:
+ return True, "valid"
+
+ # For sentences, stricter ratio
+ ratio = len_corr / len_orig if len_orig > 0 else 0
+ if ratio > 2.0 or ratio < 0.5:
+ return False, "word_count_mismatch"
+
+ return True, "valid"
+
+ def validate(self, original: str, corrected: str, error_type: str) -> Tuple[bool, str]:
+ """
+ Main validation logic
+ """
+ # 0. Sanity Check
+ if not corrected or not corrected.strip():
+ return False, "empty_output"
+
+ # ═══════════════════════════════════════════════════════════════════════════
+ # SPACE LENIENCY (Phase 1 - Solutions.md الفكرة 3)
+ # ═══════════════════════════════════════════════════════════════════════════
+ # If the ONLY difference is whitespace → accept if resulting words are valid
+ # This fixes split/merge issues like "فيالمدرسة" → "في المدرسة"
+ original_no_space = original.replace(' ', '').replace('\u200c', '') # Also handle ZWNJ
+ corrected_no_space = corrected.replace(' ', '').replace('\u200c', '')
+
+ if original_no_space == corrected_no_space:
+ # Only whitespace changed - accept immediately
+ return True, "space_leniency_accept"
+
+ # 1. Length Ratio Check
+ len_orig = len(original)
+ len_corr = len(corrected)
+
+ # Allow expansion for word splitting
+ if len_corr > len_orig * 2.5:
+ return False, "too_long"
+
+ # Allow shrinking (but not typically more than 50% unless removing repetition)
+ if len_corr < len_orig * 0.5:
+ # Exception: if original had excessive repetition
+ if error_type == ErrorType.CHAR_REPETITION:
+ pass
+ else:
+ return False, "too_short"
+
+ # 2. Check Word Count
+ is_valid_count, reason = self.check_word_count(original, corrected)
+ if not is_valid_count:
+ return False, reason
+
+ # 3. Check Character Preservation
+ # Critical for avoiding hallucinations
+ is_valid_chars, reason = self.check_character_preservation(original, corrected)
+ if not is_valid_chars:
+ # Exception: If input was garbage/keyboard mash, preservation might be low.
+ # But for valid inputs, this prevents changing "كتاب" to "مكتبة" (if no roots match)
+ return False, reason
+
+ return True, "valid"
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# VOCABULARY MANAGER (NEW - Phase 1)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class VocabularyManager:
+ """
+ Centralized vocabulary management for OOV/IV detection.
+ Key for vocabulary-aware acceptance: OOV→IV = accept, IV→OOV = reject.
+ """
+
+ # Arabic character equivalence for normalization
+ HAMZA_VARIANTS = {'أ', 'إ', 'آ', 'ء', 'ؤ', 'ئ', 'ا'}
+ ALEF_NORMALIZED = 'ا'
+ TA_MARBUTA = 'ة'
+ HA = 'ه'
+ YA_VARIANTS = {'ي', 'ى'}
+ YA_NORMALIZED = 'ي'
+
+ def __init__(self, tokenizer):
+ self.tokenizer = tokenizer
+
+ # Build vocabulary set from tokenizer (exclude subwords and short tokens)
+ self.vocab = {
+ w for w in tokenizer.get_vocab().keys()
+ if w.isalpha() and not w.startswith('##') and len(w) > 1
+ }
+
+ # Frequency rank: lower index = more common (usually)
+ self.vocab_rank = {w: i for w, i in tokenizer.get_vocab().items()}
+
+ # Build normalized vocabulary for fuzzy matching
+ self.normalized_vocab = {self.normalize_for_comparison(w): w for w in self.vocab}
+
+ print(f"📚 VocabularyManager initialized: {len(self.vocab)} words")
+
+ @classmethod
+ def normalize_for_comparison(cls, word: str) -> str:
+ """
+ Normalize Arabic word for comparison (hamza, ta marbuta, etc.)
+ Used for equivalence checking, not for final output.
+ """
+ result = []
+ for i, char in enumerate(word):
+ # Normalize Hamza variants to Alef
+ if char in cls.HAMZA_VARIANTS:
+ result.append(cls.ALEF_NORMALIZED)
+ # Normalize Ta Marbuta to Ha at word end
+ elif char == cls.TA_MARBUTA and i == len(word) - 1:
+ result.append(cls.HA)
+ # Normalize Ya variants
+ elif char in cls.YA_VARIANTS:
+ result.append(cls.YA_NORMALIZED)
+ else:
+ result.append(char)
+ return ''.join(result)
+
+ def is_iv(self, word: str) -> bool:
+ """Check if word is In-Vocabulary (known word)."""
+ clean = re.sub(r'[^\w]', '', word)
+ if not clean:
+ return True # Empty/punctuation only = treat as valid
+
+ # Direct check
+ if clean in self.vocab:
+ return True
+
+ # Normalized check (handles hamza/ta marbuta variations)
+ normalized = self.normalize_for_comparison(clean)
+ if normalized in self.normalized_vocab:
+ return True
+
+ return False
+
+ def is_oov(self, word: str) -> bool:
+ """Check if word is Out-Of-Vocabulary (unknown word)."""
+ return not self.is_iv(word)
+
+ def get_frequency_rank(self, word: str) -> int:
+ """Get frequency rank (lower = more common). Returns 999999 for OOV."""
+ clean = re.sub(r'[^\w]', '', word)
+ return self.vocab_rank.get(clean, 999999)
+
+ def all_words_iv(self, text: str) -> bool:
+ """Check if ALL words in text are In-Vocabulary."""
+ words = text.split()
+ return all(self.is_iv(w) for w in words)
+
+ def count_oov_words(self, text: str) -> int:
+ """Count number of OOV words in text."""
+ words = text.split()
+ return sum(1 for w in words if self.is_oov(w))
+
+ def get_oov_words(self, text: str) -> List[str]:
+ """Get list of OOV words in text."""
+ words = text.split()
+ return [w for w in words if self.is_oov(w)]
+
+ def words_are_equivalent(self, word1: str, word2: str) -> bool:
+ """
+ Check if two words are equivalent (considering Arabic character variations).
+ Useful for accepting corrections that only differ in hamza/ta marbuta.
+ """
+ norm1 = self.normalize_for_comparison(word1)
+ norm2 = self.normalize_for_comparison(word2)
+ return norm1 == norm2
+
+ @staticmethod
+ def damerau_levenshtein_distance(s1: str, s2: str) -> int:
+ """
+ Calculate Damerau-Levenshtein distance (transpositions count as 1 edit).
+ This is better for Arabic typos like اقصتاديا→اقتصاديا (swap صت→تص).
+ """
+ return jellyfish.damerau_levenshtein_distance(s1, s2)
+
+ def calculate_similarity(self, original: str, corrected: str) -> float:
+ """
+ Calculate similarity score using Damerau-Levenshtein distance.
+ Returns value between 0 and 1 (1 = identical).
+ """
+ dist = self.damerau_levenshtein_distance(original, corrected)
+ max_len = max(len(original), len(corrected), 1)
+ return 1.0 - (dist / max_len)
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# WORD ALIGNER (Phase 2 - Solutions.md الفكرة 5)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class WordAligner:
+ """
+ Aligns input and output words to create hybrid corrections.
+ Helps when model fixes one word but breaks another (Raw Wins/Both Wrong cause).
+ """
+
+ def __init__(self, vocab_manager):
+ """Initialize with VocabularyManager for IV checks."""
+ self.vocab = vocab_manager
+
+ def align_words(self, input_text: str, output_text: str) -> str:
+ """
+ Create hybrid by selecting best word from each position.
+ Uses simple space-based alignment (works for most Arabic cases).
+ """
+ input_words = input_text.split()
+ output_words = output_text.split()
+
+ # If lengths differ significantly, alignment is risky -> fallback to output
+ if abs(len(input_words) - len(output_words)) > 2:
+ input_oov = self.vocab.count_oov_words(input_text)
+ output_oov = self.vocab.count_oov_words(output_text)
+ return output_text if output_oov < input_oov else input_text
+
+ result = []
+
+ # Simple position-based alignment (min length)
+ min_len = min(len(input_words), len(output_words))
+
+ for i in range(min_len):
+ in_word = input_words[i]
+ out_word = output_words[i]
+
+ best_word = self._select_best_word(in_word, out_word)
+ result.append(best_word)
+
+ # Append remaining words from the longer sequence
+ if len(output_words) > min_len:
+ result.extend(output_words[min_len:])
+ elif len(input_words) > min_len:
+ # If input is longer, verify if trailing words are IV
+ # If trailing input words are OOV, maybe model was right to remove them?
+ # Safest is to keep them if they are IV, else drop.
+ for w in input_words[min_len:]:
+ if self.vocab.is_iv(w):
+ result.append(w)
+
+ return ' '.join(result)
+
+ def _select_best_word(self, input_word: str, output_word: str) -> str:
+ """
+ Select best word between input and output version.
+
+ Logic:
+ 1. Input OOV + Output IV → Take Output (Model fixed it)
+ 2. Input IV + Output OOV → Keep Input (Model broke it)
+ 3. Input IV + Output IV → Keep Input (Conservative) unless Output is much better?
+ - For now, strict conservative: if input is valid, keep it.
+ 4. Both OOV → Take Output (Model likely closer)
+ """
+ if input_word == output_word:
+ return input_word
+
+ in_iv = self.vocab.is_iv(input_word)
+ out_iv = self.vocab.is_iv(output_word)
+
+ # Case 1: Correction worked (OOV -> IV)
+ if not in_iv and out_iv:
+ return output_word
+
+ # Case 2: Correction broke it (IV -> OOV)
+ if in_iv and not out_iv:
+ return input_word
+
+ # Case 3: Both IV (Semantic change or split/merge)
+ # Conservative: Keep input to avoid semantic drift (Contextual errors are rare compared to typos)
+ if in_iv and out_iv:
+ return input_word
+
+ # Case 4: Both OOV
+ # Take output, usually closer to target even if still OOV
+ return output_word
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# SPLIT/MERGE SPECIALIST (Phase 2 - Solutions.md الفكرة 4)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class SplitMergeSpecialist:
+ """
+ Handles word splitting and merging with vocabulary validation.
+
+ Key patterns:
+ 1. SPLIT: OOV word that can be split into two IV words
+ - فيالغالب → في الغالب
+ - يقعبجماعة → يقع بجماعة
+ 2. MERGE: Adjacent OOV fragments that can merge to IV
+ - السوري ة → السورية (ta-marbuta attachment)
+ - ال كتاب → الكتاب
+ """
+
+ # Common Arabic prefixes that can be detached
+ SEPARABLE_PREFIXES = [
+ # Prepositions (longer first for greedy matching)
+ 'من', 'في', 'على', 'عن', 'مع', 'إلى', 'الى', 'حتى', 'منذ', 'خلال',
+ 'بعد', 'قبل', 'بين', 'حول', 'تحت', 'فوق', 'أمام', 'وراء', 'دون',
+ # Particles
+ 'أن', 'لن', 'لم', 'قد', 'سوف', 'كي', 'إذا', 'لو', 'مثل', 'غير',
+ # Call particle
+ 'يا',
+ ]
+
+ # Protected short words that shouldn't be split
+ PROTECTED_WORDS = {
+ 'في', 'من', 'على', 'عن', 'مع', 'إلى', 'الى', 'ان', 'أن', 'لا', 'ما', 'هو', 'هي',
+ 'لم', 'لن', 'قد', 'كل', 'كان', 'ذلك', 'هذا', 'هذه', 'التي', 'الذي', 'بين',
+ }
+
+ def __init__(self, vocab_manager):
+ """Initialize with VocabularyManager for IV checks."""
+ self.vocab = vocab_manager
+ self.separable_prefixes = sorted(
+ self.SEPARABLE_PREFIXES, key=len, reverse=True
+ )
+
+ def split_word(self, word: str) -> str:
+ """
+ Try to split an OOV word into IV components.
+
+ STRICT Strategy: Only split when BOTH parts are IV.
+ This prevents over-splitting like معظم → مع ظم
+ """
+ # Short words: don't split
+ if len(word) < 4:
+ return word
+
+ # Already IV: no need to split
+ if self.vocab.is_iv(word):
+ return word
+
+ # Protected words: don't split
+ if word in self.PROTECTED_WORDS:
+ return word
+
+ # 1. Try separable prefixes first (higher priority)
+ for prefix in self.separable_prefixes:
+ if word.startswith(prefix) and len(word) > len(prefix) + 1:
+ remainder = word[len(prefix):]
+
+ # Only accept if remainder is IV
+ if self.vocab.is_iv(remainder):
+ return f"{prefix} {remainder}"
+
+ # 2. Try all positions - STRICT: BOTH parts must be IV
+ for i in range(2, len(word) - 1):
+ left = word[:i]
+ right = word[i:]
+
+ if self.vocab.is_iv(left) and self.vocab.is_iv(right):
+ return f"{left} {right}"
+
+ # No valid split found
+ return word
+
+ def merge_fragments(self, text: str) -> str:
+ """
+ Try to merge adjacent OOV fragments into IV words.
+
+ Key patterns:
+ 1. Ta-marbuta detachment: السوري ة → السورية (Safe even if السوري is IV)
+ 2. Al- detachment: ال كتاب → الكتاب
+ 3. General OOV+OOV merging: Only if both are OOV and result is IV
+ """
+ words = text.split()
+ if len(words) < 2:
+ return text
+
+ result = []
+ i = 0
+
+ while i < len(words):
+ word = words[i]
+
+ # Try to merge with next word
+ if i + 1 < len(words):
+ next_word = words[i + 1]
+ merged = word + next_word
+
+ # Pattern 1: Detached suffix (ة، ه، ي، ك...)
+ # Allow merging even if 'word' is IV because detached suffix is definitely wrong
+ if len(next_word) == 1 and next_word in 'ةهاي':
+ if self.vocab.is_iv(merged):
+ result.append(merged)
+ i += 2
+ continue
+
+ # Pattern 2: Detached 'Al-' prefix
+ # ال كتاب → الكتاب (Safe to merge)
+ if word == 'ال' and len(next_word) >= 2:
+ if self.vocab.is_iv(merged):
+ result.append(merged)
+ i += 2
+ continue
+
+ # Pattern 3: General OOV + OOV → IV
+ # STRICT: Both must be OOV to avoid merging valid words
+ if self.vocab.is_oov(word) and self.vocab.is_oov(next_word):
+ if self.vocab.is_iv(merged):
+ result.append(merged)
+ i += 2
+ continue
+
+ # Pattern 4: Short OOV fragment (1-2 chars) merge
+ if len(word) <= 2 and self.vocab.is_oov(word):
+ if self.vocab.is_iv(merged):
+ result.append(merged)
+ i += 2
+ continue
+
+ result.append(word)
+ i += 1
+
+ return ' '.join(result)
+
+ def process_text(self, text: str) -> str:
+ """
+ Apply full split/merge processing to text.
+ Order: First merge, then split.
+ """
+ # Step 1: Merge fragments
+ text = self.merge_fragments(text)
+
+ # Step 2: Split OOV words
+ words = text.split()
+ processed = []
+
+ for word in words:
+ if self.vocab.is_oov(word) and len(word) >= 4:
+ split_result = self.split_word(word)
+ processed.append(split_result)
+ else:
+ processed.append(word)
+
+ return ' '.join(processed)
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# EDIT DISTANCE CORRECTOR (NEW!)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+
+class EditDistanceCorrector:
+ """
+ Generates candidates based on Levenshtein distance.
+ Uses BERT Vocabulary to filter for valid words.
+ """
+ def __init__(self, tokenizer):
+ self.tokenizer = tokenizer
+ # Build strict vocabulary (ignore subwords starting with ## and punctuation)
+ self.vocab = {
+ w for w in tokenizer.get_vocab().keys()
+ if w.isalpha() and not w.startswith('##') and len(w) > 1
+ }
+ # Frequency rank heuristic: lower index = higher frequency (usually)
+ self.vocab_rank = {w: i for w, i in tokenizer.get_vocab().items()}
+
+ def edits1(self, word):
+ """All edits that are one edit away from `word`."""
+ letters = 'أابتثجحخدذرزسشصضطظعغفقكلمنهويءآىةئؤ' # Arabic chars
+ splits = [(word[:i], word[i:]) for i in range(len(word) + 1)]
+ deletes = [L + R[1:] for L, R in splits if R]
+ transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
+ replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
+ inserts = [L + c + R for L, R in splits for c in letters]
+ return set(deletes + transposes + replaces + inserts)
+
+ def edits2(self, word):
+ """All edits that are two edits away from `word`."""
+ return (e2 for e1 in self.edits1(word) for e2 in self.edits1(e1))
+
+ def known(self, words):
+ """The subset of `words` that appear in the dictionary of known words."""
+ return set(w for w in words if w in self.vocab)
+
+ def generate_candidate(self, text: str) -> str:
+ """
+ Generate a candidate sentence by fixing OOV words using Edit Distance.
+ """
+ words = text.split()
+ corrected_words = []
+
+ for word in words:
+ # Clean word for checking
+ clean_word = re.sub(r'[^\w]', '', word)
+
+ # If word is known, keep it
+ if clean_word in self.vocab:
+ corrected_words.append(word)
+ continue
+
+ # If OOV, try to find neighbor
+ # 1. Edits 1
+ candidates = self.known(self.edits1(clean_word))
+
+ # 2. Edits 2 (if no Edits 1)
+ if not candidates:
+ # Optimize: Only check edits2 if word length is reasonable
+ if len(clean_word) < 7:
+ candidates = self.known(self.edits2(clean_word))
+
+ if candidates:
+ # Pick best candidate: Lowest vocab rank (most frequent)
+ best_candidate = min(candidates, key=lambda w: self.vocab_rank.get(w, 999999))
+ corrected_words.append(best_candidate)
+ else:
+ # No correction found, keep original
+ corrected_words.append(word)
+
+ return ' '.join(corrected_words)
+
+
+ @staticmethod
+ def check_word_count(original: str, corrected: str) -> Tuple[bool, str]:
+ """Check if word count is reasonable"""
+ words_original = original.split()
+ words_corrected = corrected.split()
+
+ # Allow more flexibility for merged words
+ # If original has long words (>7 chars), allow more splits
+ has_long_word = any(len(w) > 7 for w in words_original)
+ max_expansion = 3 if has_long_word else 1
+
+ if len(words_corrected) > len(words_original) + max_expansion:
+ return False, "too_many_words"
+
+ return True, "valid"
+
+ @staticmethod
+ def check_token_overlap(original: str, corrected: str) -> Tuple[bool, str]:
+ """Check token overlap"""
+ tokens_original = set(original.split())
+ tokens_corrected = set(corrected.split())
+
+ if not tokens_original:
+ return True, "valid"
+
+ # Skip for short sentences (1-2 words) where edit distance is better
+ if len(tokens_original) <= 2:
+ return True, "valid"
+
+ overlap = len(tokens_original & tokens_corrected) / len(tokens_original)
+
+ if overlap < 0.2:
+ return False, "low_token_overlap"
+
+ return True, "valid"
+
+ @staticmethod
+ def check_edit_distance_reasonable(original: str, corrected: str) -> Tuple[bool, str]:
+ """Check if edit distance is reasonable"""
+ distance = OutputValidator.calculate_edit_distance(original, corrected)
+ threshold = len(original) * 0.5 # Adaptive threshold
+
+ if distance > threshold:
+ return False, "excessive_edit_distance"
+
+ return True, "valid"
+
+ @staticmethod
+ def check_length(original: str, corrected: str) -> Tuple[bool, str]:
+ """Check if length change is reasonable"""
+ max_change = len(original) * 0.25 + 3
+
+ if abs(len(corrected) - len(original)) > max_change:
+ return False, "excessive_length_change"
+
+ return True, "valid"
+
+ @staticmethod
+ def check_repetition(corrected: str) -> Tuple[bool, str]:
+ """Check for suspicious repetitions"""
+ # Check for 4+ consecutive identical chars
+ if re.search(r'(.)\1{3,}', corrected):
+ return False, "excessive_repetition"
+
+ return True, "valid"
+
+ @staticmethod
+ def check_word_quality(corrected: str) -> Tuple[bool, str]:
+ """Check for suspicious short words"""
+ words = corrected.split()
+
+ # Count 1-char words
+ single_char_count = sum(1 for w in words if len(w) == 1)
+
+ if single_char_count > 2:
+ return False, "too_many_short_words"
+
+ return True, "valid"
+
+ @staticmethod
+ def check_word_splitting_quality(corrected: str) -> Tuple[bool, str]:
+ """Check if word splitting looks suspicious"""
+ words = corrected.split()
+
+ # Check for many 2-char words
+ two_char_count = sum(1 for w in words if len(w) == 2)
+
+ if len(words) > 3 and two_char_count > len(words) * 0.5:
+ return False, "suspicious_word_splitting"
+
+ return True, "valid"
+
+ @staticmethod
+ def check_single_chars(corrected: str) -> Tuple[bool, str]:
+ """Check for suspicious single characters"""
+ words = corrected.split()
+
+ for word in words:
+ if len(word) == 1 and word not in {'و', 'ب', 'ل', 'ك', 'ف'}:
+ return False, "suspicious_short_word"
+
+ return True, "valid"
+
+ @staticmethod
+ def check_word_preservation(original: str, corrected: str) -> Tuple[bool, str]:
+ """Check if at least some words are preserved"""
+ words_original = set(original.split())
+ words_corrected = set(corrected.split())
+
+ if len(words_original) <= 1:
+ return True, "valid"
+
+ # Skip if splitting a single merged word
+ if len(words_original) == 1 and len(words_corrected) > 1:
+ return True, "valid"
+
+ preserved = len(words_original & words_corrected) / len(words_original)
+
+ if preserved < 0.1:
+ return False, "too_few_preserved_words"
+
+ return True, "valid"
+
+ @staticmethod
+ def validate(original: str, corrected: str, error_type: str = "unknown") -> Tuple[bool, str]:
+ """
+ Validate model output
+ Returns: (is_valid, reason)
+ """
+ # Run all checks
+ checks = [
+ OutputValidator.check_character_preservation(original, corrected),
+ OutputValidator.check_word_count(original, corrected),
+ OutputValidator.check_token_overlap(original, corrected),
+ OutputValidator.check_edit_distance_reasonable(original, corrected),
+ OutputValidator.check_length(original, corrected),
+ OutputValidator.check_repetition(corrected),
+ OutputValidator.check_word_quality(corrected),
+ OutputValidator.check_word_splitting_quality(corrected),
+ OutputValidator.check_single_chars(corrected),
+ OutputValidator.check_word_preservation(original, corrected),
+ ]
+
+ for is_valid, reason in checks:
+ if not is_valid:
+ return False, reason
+
+ return True, "valid"
+
+
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# CONTEXTUAL CORRECTOR (MLM-based with Batch Scoring)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class ContextualCorrector:
+ """MLM-based contextual correction for confusion pairs"""
+
+ # Common confusion pairs in Arabic
+ CONFUSION_PAIRS = [
+ ('ض', 'ظ'), ('ذ', 'ز'), ('ث', 'س'), ('ص', 'س'),
+ ('ط', 'ت'), ('ق', 'ك'), ('ه', 'ة'), ('ا', 'ى'),
+ ('ت', 'د'), ('د', 'ض'), ('ك', 'ق'), ('غ', 'ق'),
+ ('ج', 'ش'), ('س', 'ز'), ('ف', 'ب'), ('و', 'و'), # (و, و) placeholder, maybe (و, ؤ)?
+ ('ؤ', 'و'), ('ئ', 'ي'), ('ء', 'أ'), ('إ', 'أ'),
+ ]
+
+ def __init__(self, model_name: str = 'aubmindlab/bert-base-arabertv02', cache_size: int = 10000):
+ """Initialize with BERT MLM model and LRU cache"""
+ from transformers import AutoTokenizer, AutoModelForMaskedLM
+ from functools import lru_cache
+
+ self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+ self.model = AutoModelForMaskedLM.from_pretrained(model_name)
+ self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+ self.model = self.model.to(self.device)
+ self.model.eval()
+
+ # Build confusion map
+ self.confusion_map = self._build_confusion_map()
+
+ # Stats
+ self.cache_hits = 0
+ self.cache_misses = 0
+
+ # Create LRU cache for scoring
+ self._score_cache = {}
+ self.cache_size = cache_size
+
+ # Load vocabulary for filtering
+ self.vocab = self.tokenizer.get_vocab()
+
+ def _build_confusion_map(self):
+ """Build bidirectional confusion map"""
+ confusion_map = {}
+ for char1, char2 in self.CONFUSION_PAIRS:
+ if char1 not in confusion_map:
+ confusion_map[char1] = []
+ if char2 not in confusion_map:
+ confusion_map[char2] = []
+ confusion_map[char1].append(char2)
+ confusion_map[char2].append(char1)
+ return confusion_map
+
+ def get_confusable_chars(self, char: str) -> List[str]:
+ """Get confusable characters for a given char"""
+ return self.confusion_map.get(char, [])
+
+ def generate_candidates(self, word: str) -> List[str]:
+ """Generate candidate corrections for a word"""
+ candidates = [word]
+
+ # 1. Substitute confusable chars
+ for i, char in enumerate(word):
+ confusables = self.get_confusable_chars(char)
+ for conf_char in confusables:
+ candidate = word[:i] + conf_char + word[i+1:]
+ if candidate not in candidates:
+ candidates.append(candidate)
+
+ # 2. 🆕 Remove repeated characters (deletion)
+ # Fixes: مدررسة -> مدرسة, جميلل -> جميل
+ for i in range(len(word) - 1):
+ if word[i] == word[i+1]:
+ # Remove one instance of the repeated char
+ candidate = word[:i] + word[i+1:]
+ if candidate not in candidates:
+ candidates.append(candidate)
+
+ # 3. 🆕 Edit Distance 1 Candidates (Insertions, Substitutions, Transpositions)
+ # Using a restricted set of characters to avoid explosion
+ COMMON_CHARS = 'ابتثجحخدذرزسشصضطظعغفقكلمنهويأإآءئؤةى'
+
+ # Filter candidates by vocabulary to prevent hallucinations and scoring errors
+ # Only keep candidates that are valid single tokens in the vocabulary.
+
+ # Insertions (missing char)
+ for i in range(len(word) + 1):
+ for char in COMMON_CHARS:
+ candidate = word[:i] + char + word[i:]
+ if candidate in self.vocab and candidate not in candidates:
+ candidates.append(candidate)
+
+ # Substitutions (wrong char)
+ if len(word) < 7:
+ for i in range(len(word)):
+ for char in COMMON_CHARS:
+ if char != word[i]:
+ candidate = word[:i] + char + word[i+1:]
+ if candidate in self.vocab and candidate not in candidates:
+ candidates.append(candidate)
+
+ # Deletions (extra char) - General
+ for i in range(len(word)):
+ candidate = word[:i] + word[i+1:]
+ if len(candidate) > 1:
+ # For deletions, candidate might be a valid word even if not in vocab?
+ # But to be safe and consistent with scoring, let's enforce vocab.
+ # (Note: 'جميل' IS in vocab, so it works).
+ if candidate in self.vocab and candidate not in candidates:
+ candidates.append(candidate)
+
+ return candidates
+
+ def score_with_mlm(self, text: str, position: int, word: str) -> float:
+ """Score a word in context using BERT MLM"""
+ # Check cache
+ cache_key = f"{text}|{position}|{word}"
+ if cache_key in self._score_cache:
+ self.cache_hits += 1
+ return self._score_cache[cache_key]
+
+ self.cache_misses += 1
+
+ # Create masked text
+ words = text.split()
+ if position >= len(words):
+ return 0.0
+
+ masked_words = words.copy()
+ masked_words[position] = '[MASK]'
+ masked_text = ' '.join(masked_words)
+
+ # Tokenize
+ inputs = self.tokenizer(masked_text, return_tensors='pt', padding=True, truncation=True)
+ inputs = {k: v.to(self.device) for k, v in inputs.items()}
+
+ # Get predictions
+ with torch.no_grad():
+ outputs = self.model(**inputs)
+ predictions = outputs.logits
+
+ # Find mask position
+ mask_token_index = (inputs['input_ids'] == self.tokenizer.mask_token_id).nonzero(as_tuple=True)[1]
+
+ if len(mask_token_index) == 0:
+ return 0.0
+
+ # Get probabilities for the word
+ mask_token_logits = predictions[0, mask_token_index[0], :]
+ probs = torch.softmax(mask_token_logits, dim=0)
+
+ # Get word token id
+ word_tokens = self.tokenizer.encode(word, add_special_tokens=False)
+ if not word_tokens:
+ return 0.0
+
+ word_token_id = word_tokens[0]
+ score = probs[word_token_id].item()
+
+ # Update cache (with size limit)
+ if len(self._score_cache) >= self.cache_size:
+ # Remove oldest entry (simple FIFO)
+ self._score_cache.pop(next(iter(self._score_cache)))
+
+ self._score_cache[cache_key] = score
+
+ return score
+
+ def score_candidates_batch(self, text: str, position: int, candidates: List[str]) -> dict:
+ """
+ Batch score multiple candidates (NEW - more efficient!)
+ Returns: {candidate: score}
+ """
+ scores = {}
+
+ for candidate in candidates:
+ scores[candidate] = self.score_with_mlm(text, position, candidate)
+
+ return scores
+
+ def correct_word_in_context(self, text: str, position: int, threshold: float = 0.05) -> Tuple[str, dict]:
+ """
+ Correct a word in context
+ Returns: (corrected_word, metadata)
+ """
+ words = text.split()
+ if position >= len(words):
+ return words[position] if position < len(words) else "", {}
+
+ original_word = words[position]
+
+ # Generate candidates
+ candidates = self.generate_candidates(original_word)
+
+ if len(candidates) == 1:
+ return original_word, {'candidates': 1, 'corrected': False}
+
+ # Score candidates (batch)
+ scores = self.score_candidates_batch(text, position, candidates)
+
+ # Find best candidate
+ best_word = max(scores, key=scores.get)
+ best_score = scores[best_word]
+ original_score = scores[original_word]
+
+ # Check if original is in vocabulary (Single Token)
+ # We need to know if original_score is reliable (IV) or just a prefix score (OOV)
+ orig_tokens = self.tokenizer.encode(original_word, add_special_tokens=False)
+ is_orig_iv = len(orig_tokens) == 1
+
+ # Apply correction logic
+ min_abs_improvement = 1e-4
+ is_improvement = False
+
+ if is_orig_iv:
+ # Standard relative improvement for IV words
+ if best_score > original_score * (1 + threshold) and \
+ best_score > original_score + min_abs_improvement:
+ is_improvement = True
+ else:
+ # Original is OOV/Multi-token (likely specific scoring issue with prefixes like 'ال')
+ # If best_word is IV (Single Token), effectively compare its score against an absolute threshold
+ # because original_score (prefix) is misleadingly high.
+ # We use a stricter absolute threshold for this case to avoid over-correction of valid OOVs.
+ oov_correction_threshold = 0.001 # 0.1% probability minimum (Lowered to catch 'الطقس')
+
+ # Also ensure best_word is IV (which we forced in generation, but good to be sure)
+ best_tokens = self.tokenizer.encode(best_word, add_special_tokens=False)
+ is_best_iv = len(best_tokens) == 1
+
+ if is_best_iv and best_score > oov_correction_threshold:
+ # We ignore original_score here as it's likely just P(prefix)
+ is_improvement = True
+ elif best_score > original_score * (1 + threshold):
+ # Fallback to relative check if both are OOV or logic allows
+ is_improvement = True
+
+ if best_word != original_word and is_improvement:
+ return best_word, {
+ 'candidates': len(candidates),
+ 'corrected': True,
+ 'original_score': original_score,
+ 'best_score': best_score,
+ 'improvement': best_score - original_score,
+ 'is_orig_iv': is_orig_iv
+ }
+
+ return original_word, {
+ 'candidates': len(candidates),
+ 'corrected': False,
+ 'original_score': original_score
+ }
+
+ def predict_masked_token(self, text: str, position: int, top_k: int = 5) -> List[Tuple[str, float]]:
+ """
+ Predict words for a masked position.
+ Returns: List of (word, score)
+ """
+ words = text.split()
+ if position >= len(words):
+ return []
+
+ masked_words = words.copy()
+ masked_words[position] = '[MASK]'
+ masked_text = ' '.join(masked_words)
+
+ inputs = self.tokenizer(masked_text, return_tensors='pt', padding=True, truncation=True).to(self.device)
+
+ with torch.no_grad():
+ outputs = self.model(**inputs)
+ predictions = outputs.logits
+
+ mask_token_index = (inputs['input_ids'] == self.tokenizer.mask_token_id).nonzero(as_tuple=True)[1]
+
+ if len(mask_token_index) == 0:
+ return []
+
+ mask_token_logits = predictions[0, mask_token_index[0], :]
+ probs = torch.softmax(mask_token_logits, dim=0)
+
+ top_k_weights, top_k_indices = torch.topk(probs, top_k, sorted=True)
+
+ results = []
+ for i in range(top_k):
+ token_id = top_k_indices[i].item()
+ score = top_k_weights[i].item()
+ token = self.tokenizer.decode([token_id]).strip()
+ # Filter out subwords (starting with ##) and special tokens
+ if not token.startswith("##") and token not in self.tokenizer.all_special_tokens:
+ results.append((token, score))
+
+ return results
+
+ def refine_sentence_with_mask(self, text: str, threshold: float = 0.001) -> str:
+ """
+ Refine sentence by masking weak words and predicting replacements.
+ Effectively uses BERT as a contextual dictionary.
+ """
+ words = text.split()
+ refined_words = words.copy()
+
+ for i, word in enumerate(words):
+ # 1. Check confidence
+ # We use score_with_mlm. If score is very low, it's a candidate for refinement.
+ current_score = self.score_with_mlm(text, i, word)
+
+ # If word is confident enough, skip
+ # Threshold needs to be tuned. 0.001 implies 0.1% probability.
+ if current_score > threshold:
+ continue
+
+ # 2. Mask and Predict
+ predictions = self.predict_masked_token(text, i, top_k=10)
+
+ # 3. Filter and Select
+ for pred_word, pred_score in predictions:
+ if pred_word == word:
+ continue
+
+ # STRICTER CONSTRAINTS
+
+ # 1. Length Check
+ if abs(len(pred_word) - len(word)) > 2:
+ continue
+
+ # 2. Similarity Check (Edit Distance)
+ # We want to fix TYPOS, not semantic hallucinations.
+ # So the replacement MUST be structurally similar.
+ # Exception: If original word is very short (<3 chars), strict check.
+
+ dist = Levenshtein.distance(word, pred_word)
+ max_len = max(len(word), len(pred_word))
+ similarity = 1.0 - (dist / max_len)
+
+
+ # Minimum similarity required: 0.7 (Much stricter to prevent semantic shift)
+ # 'بسرعّة' -> 'بسرعة' (High sim)
+ # 'الإسلامية' -> 'التطبيقية' (Low sim - rejected)
+ if similarity < 0.7:
+ continue
+
+ # 3. Score Improvement
+ # If current word is total garbage (score < 1e-5), take any reasonable predicted word.
+ # If current word has some confidence, only replace if prediction is MUCH better.
+
+ # Check if original word is IV and common
+ is_original_common = current_score > 0.001
+
+ if is_original_common:
+ # Very strict if original seems okay
+ if pred_score > current_score * 500:
+ refined_words[i] = pred_word
+ break
+ else:
+ # Looser if original is weak
+ if pred_score > current_score * 10 or pred_score > 0.1:
+ refined_words[i] = pred_word
+ break # Take the top valid prediction
+
+ return ' '.join(refined_words)
+
+ def calculate_sentence_score(self, text: str) -> float:
+ """
+ Calculate a 'fluency' score for the sentence using BERT MLM.
+ Returns the average probability of each word being predicted in its context.
+ """
+ words = text.split()
+ if not words:
+ return 0.0
+
+ total_score = 0.0
+ scored_words = 0
+
+ for i, word in enumerate(words):
+ # We skip scoring very common stopwords to focus on content/structure?
+ # No, keep it simple for now: score everything.
+ score = self.score_with_mlm(text, i, word)
+ total_score += score
+ scored_words += 1
+
+ if scored_words == 0:
+ return 0.0
+
+ return total_score / scored_words
+
+ def correct_sentence(self, text: str, threshold: float = 0.01) -> Tuple[str, dict]:
+ """
+ Correct all words in a sentence
+ Returns: (corrected_text, metadata)
+ """
+ words = text.split()
+ corrected_words = []
+ corrections = []
+
+ for i, word in enumerate(words):
+ corrected_word, meta = self.correct_word_in_context(text, i, threshold)
+ corrected_words.append(corrected_word)
+
+ if meta.get('corrected'):
+ corrections.append({
+ 'position': i,
+ 'original': word,
+ 'corrected': corrected_word,
+ 'confidence': meta.get('improvement', 0)
+ })
+
+ corrected_text = ' '.join(corrected_words)
+
+ return corrected_text, {
+ 'corrections': corrections,
+ 'total_words': len(words)
+ }
+
+
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# MAIN SPELL CHECKER CLASS
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class ArabicSpellChecker:
+ """Main Arabic Spell Checker class"""
+
+ def __init__(self, model, tokenizer, device, use_contextual: bool = True):
+ """Initialize spell checker with model and components"""
+ self.model = model
+ self.tokenizer = tokenizer
+ self.device = device
+
+ # Initialize components
+ self.postprocessor = AraSpellPostProcessor()
+ self.classifier = ErrorClassifier()
+ self.rules = RulesBasedCorrector()
+ self.validator = OutputValidator()
+ self.vocab_manager = VocabularyManager(tokenizer) # Phase 1: Vocabulary Manager
+ self.edit_corrector = EditDistanceCorrector(tokenizer) # Edit Distance candidates
+ self.split_merge = SplitMergeSpecialist(self.vocab_manager) # Phase 2: Split/Merge
+
+ # Phase 2: WordAligner for word-level hybrid corrections
+ self.word_aligner = WordAligner(self.vocab_manager)
+
+ # Initialize contextual corrector (optional)
+ self.use_contextual = use_contextual
+ if use_contextual:
+ try:
+ # Initialize ContextualCorrector
+ self.contextual = ContextualCorrector()
+ print("✅ Contextual correction enabled")
+ except Exception as e:
+ print(f"⚠️ Contextual correction disabled: {e}")
+ self.contextual = None
+ self.use_contextual = False
+ else:
+ self.contextual = None
+ def _fix_repeated_end_chars(self, text: str) -> str:
+ """
+ 🆕 Fix repeated characters at word endings
+
+ Examples:
+ اليومم → اليوم
+ جميلل → جميل
+ صباحح → صباح
+ """
+ # Remove repeated chars at word end (keep only one)
+ text = re.sub(r'([ا-ي])\1+\b', r'\1', text)
+ return text
+
+ def _fix_merged_with_errors(self, text: str) -> str:
+ """
+ 🆕 Fix merged words that contain errors
+
+ Examples:
+ الممدرسة → المدرسة
+ الكتابب → الكتاب
+ الططالب → الطالب
+ """
+ # Pattern 1: ال + repeated char + word
+ text = re.sub(r'ال([ا-ي])\1+([ا-ي]{2,})', r'ال\2', text)
+
+ # Pattern 2: word + repeated char at end
+ text = re.sub(r'\b([ا-ي]{3,})([ا-ي])\2+\b', r'\1\2', text)
+
+ return text
+
+
+ def _split_merged_words_linguistic(self, text: str) -> str:
+ """
+ 🆕 Split merged words using linguistic patterns
+
+ Examples:
+ كلصباح → كل صباح
+ فيالطريق → في الطريق
+ السلامعليكم → السلام عليكم
+ """
+ # Pattern 1: Prepositions + (article)? + word
+ # Added: ك (like in كالكتاب) but careful not to split overlapping words
+ text = re.sub(
+ r'\b(في|من|إلى|الى|حتى|منذ|خلال|بعد|قبل)(ال)?([ا-ي]{3,})',
+ r'\1 \2\3',
+ text
+ )
+
+ # Pattern 2: كل + word
+ text = re.sub(r'\b(كل)([ا-ي]{3,})', r'\1 \2', text)
+
+ # Pattern 3: Article repetition
+ text = re.sub(r'([ا-ي]{3,})(ال)([ا-ي]{3,})', r'\1 \2\3', text)
+
+ # Pattern 4: Single-letter prepositions
+ text = re.sub(r'\b([بلك])(ال)?([ا-ي]{3,})', r'\1 \2\3', text)
+
+ # Pattern 5: Word + عليكم/عليك
+ text = re.sub(r'([ا-ي]{4,})(عليكم|عليك|عليه|عليها)', r'\1 \2', text)
+
+ # Pattern 6: على/عن in middle of (merged) words
+ text = re.sub(r'([ا-ي]{3,})(على|عن)([ا-ي]{3,})', r'\1 \2 \3', text)
+
+ # Pattern 7: بسم الله الرحمن الرحيم (common concatenation)
+ text = re.sub(r'\bبسماللهالرحمنالرحيم\b', 'بسم الله الرحمن الرحيم', text)
+ text = re.sub(r'\bبسمالله\b', 'بسم الله', text)
+ text = re.sub(r'اللهالرحمن', 'الله الرحمن', text)
+ text = re.sub(r'الرحمنالرحيم', 'الرحمن الرحيم', text)
+
+ return text
+
+ def _split_long_words_heuristic(self, text: str, max_length: int = 15) -> str:
+ """
+ 🆕 Split suspiciously long words using heuristics
+ """
+ words = text.split()
+ result = []
+
+ for word in words:
+ if len(word) <= max_length:
+ result.append(word)
+ continue
+
+ # Check for embedded article
+ if 'ال' in word[2:]:
+ parts = word.split('ال', 1)
+ if len(parts[0]) >= 2 and len(parts[1]) >= 3:
+ result.extend([parts[0], 'ال' + parts[1]])
+ continue
+
+ # Check for common prefixes at start of long word
+ if len(word) >= 8:
+ split_found = False
+ for split_pos in [2, 3]:
+ prefix = word[:split_pos]
+ suffix = word[split_pos:]
+
+ if prefix in ['في', 'من', 'على', 'عن', 'مع', 'كل', 'ب', 'ل', 'ك']:
+ result.extend([prefix, suffix])
+ split_found = True
+ break
+
+ if not split_found:
+ result.append(word)
+ else:
+ result.append(word)
+
+ return ' '.join(result)
+
+ def _normalize_tanween_patterns(self, text: str) -> str:
+ """
+ 🆕 Normalize tanween patterns
+
+ Examples:
+ جدأ → جداً
+ كثيرأ → كثيراً
+ """
+ # أ at word end → اً
+ text = re.sub(r'([ا-ي]{2,})أ\b', r'\1اً', text)
+
+ # Remove standalone أ
+ text = re.sub(r'\s+أ\s+', ' ', text)
+
+ # Fix accidental splits (e.g. ب + space + word)
+ text = re.sub(r'\b([بلك])\s+([ا-ي])', r'\1\2', text)
+
+ return text
+
+
+
+
+
+ def preprocess(self, text: str) -> str:
+ """Preprocessing pipeline (مع التحسينات المدمجة)"""
+ # Basic normalization
+ text = self.postprocessor.remove_harakat(text)
+ text = self.postprocessor.remove_tatweel(text)
+ text = self.postprocessor.normalize_special_chars(text)
+
+ # 🆕 التحسينات المدمجة (IMPROVEMENTS INTEGRATED!)
+ # Fix repeated chars and merged words with errors FIRST
+ text = self._fix_repeated_end_chars(text)
+ text = self._fix_merged_with_errors(text)
+
+ # Then split merged words
+ text = self._split_merged_words_linguistic(text)
+ text = self._split_long_words_heuristic(text)
+ text = self._normalize_tanween_patterns(text)
+
+ # Merge separated 'ال'
+ text = self.postprocessor.merge_separated_al(text)
+
+ # Collapse repetitions
+ text = self.postprocessor.unified_collapse_repeated(text)
+
+ # Rules-based fixes
+ text = self.rules.fix_char_substitution(text)
+ text = self.rules.fix_char_repetition(text)
+
+ # Normalize spaces
+ text = self.postprocessor.normalize_spaces(text)
+
+ return text
+
+ def _fix_word_split(self, text: str) -> str:
+ """Fix over-split words by joining fragments"""
+ return self.postprocessor.join_fragments(text)
+
+ def postprocess(self, text: str, original: str = "") -> str:
+ """Postprocessing pipeline"""
+ return self.postprocessor.full_postprocess(text, original)
+
+ def model_inference(self, text: str, num_return_sequences: int = 5) -> List[str]:
+ """Run seq2seq model inference and return top candidates"""
+ # Tokenize
+ inputs = self.tokenizer(text, return_tensors='pt', padding=True, truncation=True, max_length=128)
+ inputs = {k: v.to(self.device) for k, v in inputs.items()}
+
+ # Generate (encoder-decoder needs decoder_start_token_id / bos_token_id)
+ decoder_start = getattr(
+ self.model.config, 'decoder_start_token_id', None
+ ) or getattr(self.model.config, 'bos_token_id', None) or self.tokenizer.cls_token_id
+ pad_id = getattr(self.model.config, 'pad_token_id', None) or self.tokenizer.pad_token_id
+ with torch.no_grad():
+ outputs = self.model.generate(
+ **inputs,
+ max_length=128,
+ num_beams=5,
+ num_return_sequences=num_return_sequences,
+ early_stopping=True,
+ decoder_start_token_id=decoder_start,
+ pad_token_id=pad_id,
+ )
+
+ # Decode
+ candidates = self.tokenizer.batch_decode(outputs, skip_special_tokens=True)
+
+ return candidates
+
+ def correct(self, text: str) -> str:
+ """
+ Main correction pipeline (RERANKING APPROACH)
+
+ Steps:
+ 1. Preprocess
+ 2. Generate Candidates (Model Beams + Baseline)
+ 3. Rerank Candidates (Validator + Fluency)
+ 4. Select Best
+ 5. Postprocess
+ """
+ if not text or not text.strip():
+ return text
+
+ original = text
+
+ # 1. Preprocess
+ # This provides a strong baseline candidate
+ preprocessed_text = self.preprocess(text)
+
+ # 2. Classify error type
+ error_type = self.classifier.classify(preprocessed_text)
+
+ # 3. Generate Candidates
+ candidates = []
+
+ # A. Baseline (Preprocessed)
+ candidates.append(preprocessed_text)
+
+ # B. Smart Rules Candidate (Aggressive Heuristic)
+ # This helps when the model fails but the rule-based fix is obvious
+ rules_candidate = self.rules.advanced_heuristic_repair(text)
+ candidates.append(rules_candidate)
+
+ # B2. Edit Distance Candidate (NEW!)
+ # Tries to fix typos using simple edit distance (Norvig)
+ edit_candidate = self.edit_corrector.generate_candidate(text)
+ if edit_candidate != text and edit_candidate != rules_candidate:
+ candidates.append(edit_candidate)
+
+ # B3. Split/Merge Candidate (Phase 2)
+ # NOTE: Disabled - caused regression (Hybrid Wins 144→134)
+ # split_merge_candidate = self.split_merge.process_text(preprocessed_text)
+ # if split_merge_candidate != preprocessed_text and split_merge_candidate not in candidates:
+ # candidates.append(split_merge_candidate)
+
+ # C. Model Beams
+ try:
+ model_candidates = self.model_inference(preprocessed_text, num_return_sequences=5)
+ candidates.extend(model_candidates)
+
+ # D. Word-Aligned Hybrid Candidate (Phase 2 - Solution 5)
+ # Creates a hybrid by selecting best word from each position
+ # (OOV input + IV output → take output, IV input + OOV output → keep input)
+ if model_candidates:
+ hybrid_candidate = self.word_aligner.align_words(preprocessed_text, model_candidates[0])
+ if hybrid_candidate not in candidates:
+ candidates.append(hybrid_candidate)
+ except Exception as e:
+ print(f"⚠️ Model inference failed: {e}")
+
+ # Remove duplicates while preserving order
+ unique_candidates = []
+ seen = set()
+ for c in candidates:
+ if c not in seen:
+ unique_candidates.append(c)
+ seen.add(c)
+ candidates = unique_candidates
+
+ # 4. Rerank Candidates
+ best_candidate = preprocessed_text
+ best_score = -1.0
+
+ # Debug info
+ candidate_scores = []
+
+ for cand in candidates:
+ # A. Validation Score (Hard Penalty)
+ # Check validity against strict original
+ is_valid, reason = self.validator.validate(original, cand, error_type.value)
+
+ # Additional check: If candidate is suspiciously shorter than original (and not just harakat removal)
+ if len(cand) < len(original) * 0.5:
+ is_valid = False
+ reason = "too_short"
+
+ # ═══════════════════════════════════════════════════════════════════════════
+ # NEW: VOCABULARY-AWARE ACCEPTANCE (Phase 1 - Key Fix for Raw Wins)
+ # ═══════════════════════════════════════════════════════════════════════════
+ # Logic: OOV→IV = ACCEPT (boost), IV→OOV = REJECT (penalize)
+ # This prevents over-conservative validation from rejecting correct corrections
+
+ input_oov_count = self.vocab_manager.count_oov_words(original)
+ cand_oov_count = self.vocab_manager.count_oov_words(cand)
+
+ vocab_boost = 1.0
+
+ # Case 1: OOV→IV (Correction fixed unknown words) → Accept more readily
+ if input_oov_count > 0 and cand_oov_count < input_oov_count:
+ # Significant boost for reducing OOV words
+ oov_reduction = input_oov_count - cand_oov_count
+ vocab_boost = 1.0 + (oov_reduction * 0.3) # +30% per OOV fixed
+
+ # If ALL words are now IV, accept even with higher edit distance
+ if cand_oov_count == 0 and self.vocab_manager.all_words_iv(cand):
+ # Override validation rejection if OOV→IV
+ if not is_valid and reason not in ["empty_output"]:
+ is_valid = True
+ reason = "vocab_aware_accept"
+
+ # Case 2: IV→OOV (Correction introduced unknown words) → Penalize
+ elif cand_oov_count > input_oov_count:
+ # Penalize for introducing new OOV words
+ vocab_boost = 0.5 # 50% penalty
+
+ # Case 3: All IV to begin with → Standard validation
+ elif input_oov_count == 0 and cand_oov_count == 0:
+ # Both are valid vocab, prefer minimal edits
+ vocab_boost = 1.0
+
+ # ═══════════════════════════════════════════════════════════════════════════
+
+
+ # Penalty factor
+ # Valid: 1.0
+ # Invalid: 0.01 (Heavy penalty, essentially disqualified unless all are invalid)
+ validity_factor = 1.0 if is_valid else 0.001
+
+ # B. Fluency Score (BERT MLM)
+ fluency_score = 0.0
+ if self.use_contextual and self.contextual:
+ try:
+ fluency_score = self.contextual.calculate_sentence_score(cand)
+ except Exception as e:
+ print(f"⚠️ Scoring failed: {e}")
+ fluency_score = 0.5 # Default fallback
+ else:
+ fluency_score = 1.0
+
+ # C. Similarity Score (Damerau-Levenshtein Distance - Phase 1 improvement)
+ # Penalize unnecessary changes. Using DL distance: transpositions = 1 edit (not 2)
+ # This helps cases like اقصتاديا→اقتصاديا (swap صت→تص counts as 1)
+ dist = VocabularyManager.damerau_levenshtein_distance(preprocessed_text, cand)
+ # Using preprocessed_text as anchor because it has basic normalization.
+ # Comparison with 'original' might penalize fixing harakat/spelling.
+
+ max_len = max(len(preprocessed_text), len(cand), 1)
+ similarity = 1.0 - (dist / max_len)
+
+ # Boost matches
+ if cand == preprocessed_text:
+ similarity = 1.0
+
+ # NEW: HIGH CONFIDENCE GATING (Phase 1/3 - Solution)
+ # If model is extremely confident (high fluency) and words are valid, relax validation
+ # This allows correcting severe corruptions that fail strict edit distance
+ if fluency_score > 0.85 and cand_oov_count == 0:
+ if not is_valid and reason in ["too_short", "low_character_similarity", "word_count_mismatch"]:
+ # Check if it makes sense length-wise (don't allow completely empty or massive hallucinations)
+ if len(cand) >= len(original) * 0.4:
+ is_valid = True
+ reason = "high_confidence_override"
+ vocab_boost *= 1.2 # Bonus for high confidence
+ validity_factor = 1.0 # Reset validity factor
+
+ # Final Score
+ # Fluency is roughly [0, 1] (prob). Similarity [0, 1].
+ # We want to balance staying close to original vs being fluent.
+ # If fluency is very low, it's garbage.
+ # If similarity is very low, it's hallucination.
+
+ # Weighting:
+ # We value Similarity highly to be conservative.
+ # But we need Fluency to break ties or fix errors.
+
+ # New Formula:
+ # Score = (Fluency^0.3) * (Similarity^2.0) * Validity * VocabBoost
+ # Using exponent to control trade-off.
+ # High sim power -> prefers closer matches.
+ # Low fluency power -> flattens probability differences (since probs are small).
+ # VocabBoost: rewards OOV→IV, penalizes IV→OOV
+
+ final_score = (fluency_score ** 0.3) * (similarity ** 3.0) * validity_factor * vocab_boost
+
+ candidate_scores.append({
+ 'text': cand,
+ 'is_valid': is_valid,
+ 'reason': reason,
+ 'fluency': fluency_score,
+ 'similarity': similarity,
+ 'vocab_boost': vocab_boost, # NEW: Track vocab boost
+ 'input_oov': input_oov_count,
+ 'cand_oov': cand_oov_count,
+ 'final_score': final_score
+ })
+
+ if final_score > best_score:
+ best_score = final_score
+ best_candidate = cand
+
+ # 5. Postprocess Winner
+ result = self.postprocess(best_candidate, original)
+
+ # 6. Contextual fine-tuning (BERT Masked Refinement)
+ # Note: Applying to full sentence (OOV-only mode caused regression due to lack of context)
+ if self.use_contextual and self.contextual:
+ if len(result) > 3:
+ result = self.contextual.refine_sentence_with_mask(result)
+
+ # 7. Phase 2: Safe Split/Merge Post-processing
+ # Only apply merge_fragments (safe: only merges when result is IV)
+ # This fixes ta-marbuta detachment like السوري ة → السورية
+ result = self.split_merge.merge_fragments(result)
+
+ return result
+
+
+print("✅ All classes defined successfully!")
+print(" - ErrorType")
+print(" - AraSpellPostProcessor")
+print(" - ErrorClassifier")
+print(" - RulesBasedCorrector")
+print(" - OutputValidator")
+print(" - ContextualCorrector")
+print(" - ArabicSpellChecker")
+
+
+print("✅ AraSpell classes loaded successfully")
+
diff --git a/src/css/base.css b/src/css/base.css
new file mode 100644
index 0000000000000000000000000000000000000000..981586b4c05ef8c5455d90819ad439412c74c53b
--- /dev/null
+++ b/src/css/base.css
@@ -0,0 +1,184 @@
+/* Bayan Base Styles — typography & global */
+
+*,
+*::before,
+*::after {
+ box-sizing: border-box;
+}
+
+html {
+ font-family: var(--font-family-primary);
+}
+
+body {
+ font-family: var(--font-family-primary);
+ font-size: var(--font-size-body);
+ font-weight: var(--font-weight-regular);
+ line-height: var(--line-height-body);
+ background-color: var(--color-bg);
+ color: var(--color-text-primary);
+ margin: 0;
+ -webkit-font-smoothing: antialiased;
+ -moz-osx-font-smoothing: grayscale;
+ text-rendering: optimizeLegibility;
+}
+
+/* ── Type scale ── */
+.text-display {
+ font-size: var(--font-size-display);
+ font-weight: var(--font-weight-bold);
+ line-height: 1.3;
+}
+
+.text-h1 {
+ font-size: var(--font-size-h1);
+ font-weight: var(--font-weight-bold);
+ line-height: 1.35;
+}
+
+.text-h2 {
+ font-size: var(--font-size-h2);
+ font-weight: var(--font-weight-semibold);
+ line-height: 1.4;
+}
+
+.text-h3 {
+ font-size: var(--font-size-h3);
+ font-weight: var(--font-weight-semibold);
+ line-height: 1.5;
+}
+
+.text-body {
+ font-size: var(--font-size-body);
+ font-weight: var(--font-weight-regular);
+ line-height: var(--line-height-body);
+}
+
+.text-caption {
+ font-size: var(--font-size-caption);
+ font-weight: var(--font-weight-medium);
+ line-height: 1.6;
+}
+
+.text-label {
+ font-size: var(--font-size-label);
+ font-weight: var(--font-weight-semibold);
+ line-height: 1.5;
+ letter-spacing: 0.04em;
+}
+
+/* ── Semantic text colors ── */
+.text-primary { color: var(--color-text-primary); }
+.text-secondary { color: var(--color-text-secondary); }
+.text-muted { color: var(--color-text-muted); }
+.text-brand { color: var(--color-primary); }
+.text-success { color: var(--color-success); }
+.text-warning { color: var(--color-warning); }
+.text-error { color: var(--color-error); }
+.text-secondary-brand { color: var(--color-secondary); }
+
+/* ── Page utilities ── */
+.page-bg { background-color: var(--color-bg); }
+.surface-bg { background-color: var(--color-surface); }
+
+.sr-only {
+ position: absolute;
+ width: 1px;
+ height: 1px;
+ padding: 0;
+ margin: -1px;
+ overflow: hidden;
+ clip: rect(0, 0, 0, 0);
+ white-space: nowrap;
+ border: 0;
+}
+
+.page { display: none; }
+.page.active { display: block; }
+
+.gradient-bg {
+ background: linear-gradient(
+ 165deg,
+ var(--color-bg) 0%,
+ var(--color-surface) 50%,
+ var(--color-bg) 100%
+ );
+}
+
+.gradient-accent {
+ background: linear-gradient(
+ 135deg,
+ var(--color-primary) 0%,
+ var(--color-secondary) 100%
+ );
+}
+
+.text-gradient {
+ background: linear-gradient(
+ 135deg,
+ var(--color-primary) 0%,
+ var(--color-secondary) 100%
+ );
+ -webkit-background-clip: text;
+ -webkit-text-fill-color: transparent;
+ background-clip: text;
+}
+
+.editor-content {
+ font-size: var(--font-size-editor);
+ line-height: var(--line-height-editor);
+ letter-spacing: var(--letter-spacing-arabic);
+}
+
+/* ── Focus ── */
+button:focus-visible,
+a:focus-visible,
+[contenteditable]:focus-visible,
+.suggestion-card:focus-visible,
+input:focus-visible {
+ outline: 2px solid var(--color-accent);
+ outline-offset: 2px;
+}
+
+/* ── Form controls ── */
+input[type="checkbox"],
+input[type="range"] {
+ accent-color: var(--color-primary);
+}
+
+/* ── Motion ── */
+@media (prefers-reduced-motion: reduce) {
+ *,
+ *::before,
+ *::after {
+ animation-duration: 0.01ms !important;
+ animation-iteration-count: 1 !important;
+ transition-duration: 0.01ms !important;
+ }
+
+ .card-hover:hover {
+ transform: none;
+ }
+}
+
+@keyframes fadeIn {
+ from { opacity: 0; transform: translateY(12px); }
+ to { opacity: 1; transform: translateY(0); }
+}
+
+@keyframes float {
+ 0%, 100% { transform: translateY(0); }
+ 50% { transform: translateY(-12px); }
+}
+
+@keyframes spin {
+ to { transform: rotate(360deg); }
+}
+
+@keyframes pulse-subtle {
+ 0%, 100% { opacity: 1; }
+ 50% { opacity: 0.55; }
+}
+
+.animate-fade-in { animation: fadeIn 0.5s ease-out; }
+.animate-float { animation: float 6s ease-in-out infinite; }
diff --git a/src/css/components.css b/src/css/components.css
new file mode 100644
index 0000000000000000000000000000000000000000..25de441e4cddc5a40b235747ad7a997b268be5bc
--- /dev/null
+++ b/src/css/components.css
@@ -0,0 +1,1287 @@
+/* Bayan UI Components — token-driven only */
+
+/* ── Navigation ── */
+.site-nav {
+ background: var(--color-nav-bg);
+ border-bottom: 1px solid var(--color-border);
+ backdrop-filter: blur(12px);
+}
+
+.nav-link {
+ position: relative;
+ transition: color var(--transition-base);
+ color: var(--color-text-secondary);
+ background: none;
+ border: none;
+ cursor: pointer;
+ font-family: inherit;
+}
+
+.nav-link:hover,
+.nav-link.active {
+ color: var(--color-text-primary);
+}
+
+.nav-link::after {
+ content: '';
+ position: absolute;
+ bottom: -4px;
+ right: 0;
+ width: 0;
+ height: 2px;
+ background: var(--color-primary);
+ transition: width var(--transition-base);
+}
+
+.nav-link:hover::after,
+.nav-link.active::after {
+ width: 100%;
+}
+
+.nav-link-external {
+ display: inline-flex;
+ align-items: center;
+ gap: 0.35rem;
+ text-decoration: none;
+}
+
+.nav-link-external::after {
+ content: '↗';
+ font-size: 0.85em;
+ opacity: 0.7;
+ position: static;
+ width: auto;
+ height: auto;
+ background: none;
+}
+
+.mobile-drawer-link-external {
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+ text-decoration: none;
+ color: inherit;
+}
+
+.mobile-drawer-link-external::after {
+ content: '↗';
+ opacity: 0.65;
+ font-size: 0.9em;
+}
+
+.footer-external-link {
+ color: var(--color-text-secondary);
+}
+
+.footer-external-link:hover {
+ color: var(--color-primary);
+}
+
+.theme-toggle {
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ width: 44px;
+ height: 44px;
+ border-radius: var(--radius-md);
+ border: 1px solid var(--color-border);
+ background: var(--color-surface);
+ color: var(--color-text-primary);
+ cursor: pointer;
+ transition: background var(--transition-base), border-color var(--transition-base);
+}
+
+.theme-toggle:hover {
+ background: var(--color-surface-elevated);
+ border-color: var(--color-border-strong);
+}
+
+.mobile-menu-btn {
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ width: 44px;
+ height: 44px;
+ border-radius: var(--radius-md);
+ border: 1px solid var(--color-border);
+ background: var(--color-surface);
+ color: var(--color-text-primary);
+ cursor: pointer;
+ transition: background var(--transition-base);
+}
+
+.mobile-menu-btn:hover {
+ background: var(--color-surface-elevated);
+}
+
+.mobile-drawer {
+ position: fixed;
+ inset: 0;
+ z-index: 100;
+ pointer-events: none;
+ visibility: hidden;
+}
+
+.mobile-drawer.open {
+ pointer-events: auto;
+ visibility: visible;
+}
+
+.mobile-drawer-backdrop {
+ position: absolute;
+ inset: 0;
+ background: var(--color-overlay);
+ opacity: 0;
+ transition: opacity var(--transition-slow);
+}
+
+.mobile-drawer.open .mobile-drawer-backdrop {
+ opacity: 1;
+}
+
+.mobile-drawer-panel {
+ position: absolute;
+ top: 0;
+ right: 0;
+ width: min(280px, 85vw);
+ height: 100%;
+ background: var(--color-surface);
+ border-left: 1px solid var(--color-border);
+ padding: var(--spacing-lg);
+ transform: translateX(100%);
+ transition: transform var(--transition-slow);
+ display: flex;
+ flex-direction: column;
+ gap: var(--spacing-sm);
+}
+
+.mobile-drawer.open .mobile-drawer-panel {
+ transform: translateX(0);
+}
+
+.mobile-drawer-link {
+ display: block;
+ padding: var(--spacing-md);
+ border-radius: var(--radius-md);
+ color: var(--color-text-primary);
+ font-weight: var(--font-weight-semibold);
+ text-align: right;
+ border: none;
+ background: transparent;
+ cursor: pointer;
+ width: 100%;
+ font-size: var(--font-size-body);
+ font-family: inherit;
+ transition: background var(--transition-base), color var(--transition-base);
+}
+
+.mobile-drawer-link:hover,
+.mobile-drawer-link.active {
+ background: var(--color-surface-elevated);
+ color: var(--color-primary);
+}
+
+/* ── Cards ── */
+.surface-card {
+ background: var(--color-surface);
+ border: 1px solid var(--color-border);
+ border-radius: var(--radius-card);
+ box-shadow: var(--shadow-card);
+}
+
+.card-hover {
+ transition: transform var(--transition-base), box-shadow var(--transition-base);
+}
+
+.card-hover:hover {
+ transform: translateY(-3px);
+ box-shadow: var(--shadow-popover);
+}
+
+.feature-icon {
+ width: 48px;
+ height: 48px;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ border-radius: var(--radius-md);
+ font-size: 1.5rem;
+}
+
+.feature-icon--spelling {
+ background: var(--color-badge-spelling-bg);
+ color: var(--color-error);
+}
+
+.feature-icon--grammar {
+ background: var(--color-badge-grammar-bg);
+ color: var(--color-warning);
+}
+
+.feature-icon--punctuation {
+ background: var(--color-badge-punctuation-bg);
+ color: var(--color-success);
+}
+
+.feature-icon--summarize {
+ background: var(--color-summary-accent-bg);
+ color: var(--color-secondary);
+}
+
+.pricing-badge {
+ position: absolute;
+ top: -12px;
+ right: 50%;
+ transform: translateX(50%);
+ padding: 4px 16px;
+ border-radius: 20px;
+ font-size: var(--font-size-label);
+ font-weight: var(--font-weight-bold);
+}
+
+/* ── Editor shell ── */
+.editor-shell {
+ background: var(--color-surface);
+ border: 1px solid var(--color-border);
+ border-radius: var(--radius-card);
+ overflow: hidden;
+ box-shadow: var(--shadow-card);
+}
+
+.editor-toolbar {
+ display: flex;
+ flex-wrap: wrap;
+ align-items: center;
+ justify-content: space-between;
+ gap: var(--spacing-sm);
+ padding: var(--spacing-md);
+ border-bottom: 1px solid var(--color-border);
+ background: var(--color-surface);
+}
+
+.editor-tab {
+ padding: var(--spacing-sm) var(--spacing-md);
+ border-radius: var(--radius-md);
+ font-weight: var(--font-weight-bold);
+ font-size: var(--font-size-caption);
+ border: none;
+ cursor: pointer;
+ font-family: inherit;
+ transition: background var(--transition-base), color var(--transition-base);
+ background: transparent;
+ color: var(--color-text-secondary);
+ min-height: 44px;
+}
+
+.editor-tab:hover {
+ color: var(--color-text-primary);
+ background: var(--color-surface-elevated);
+}
+
+.editor-tab.active {
+ background: linear-gradient(135deg, var(--color-primary), var(--color-secondary));
+ color: var(--color-text-inverse);
+}
+
+.editor-surface {
+ background: var(--color-editor);
+ color: var(--color-text-primary);
+ min-height: 50vh;
+ padding: var(--spacing-lg);
+ font-size: var(--font-size-editor);
+ font-weight: var(--font-weight-regular);
+ line-height: var(--line-height-editor);
+ letter-spacing: var(--letter-spacing-arabic);
+ direction: rtl;
+ text-align: right;
+ white-space: pre-wrap;
+ word-wrap: break-word;
+ overflow-y: auto;
+ border-radius: var(--radius-md);
+ margin: var(--spacing-md);
+ border: 1px solid var(--color-border);
+ box-shadow: var(--shadow-editor);
+ transition: box-shadow var(--transition-base), border-color var(--transition-base);
+}
+
+@media (min-width: 768px) {
+ .editor-surface { min-height: 60vh; }
+}
+
+@media (min-width: 1024px) {
+ .editor-surface { min-height: 500px; }
+}
+
+.editor-surface:focus {
+ outline: none;
+ border-color: var(--color-border-strong);
+ box-shadow: var(--shadow-editor), 0 0 0 3px var(--focus-ring);
+}
+
+.editor-surface[data-empty="true"]::before {
+ content: attr(data-placeholder);
+ color: var(--color-placeholder);
+ font-style: italic;
+ pointer-events: none;
+ display: block;
+}
+
+.editor-surface.analyzing {
+ opacity: 0.88;
+}
+
+.editor-footer {
+ display: flex;
+ flex-wrap: wrap;
+ align-items: center;
+ justify-content: space-between;
+ gap: var(--spacing-md);
+ padding: var(--spacing-md);
+ border-top: 1px solid var(--color-border);
+ background: var(--color-surface);
+}
+
+.editor-stats {
+ display: flex;
+ flex-wrap: wrap;
+ gap: var(--spacing-md);
+ align-items: center;
+}
+
+.stat-dot {
+ width: 10px;
+ height: 10px;
+ border-radius: 50%;
+ flex-shrink: 0;
+}
+
+.stat-dot--spelling { background: var(--color-error); }
+.stat-dot--grammar { background: var(--color-warning); }
+.stat-dot--punctuation { background: var(--color-success); }
+
+.editor-actions {
+ display: flex;
+ flex-wrap: wrap;
+ gap: var(--spacing-sm);
+}
+
+/* ── Buttons ── */
+.btn-ghost {
+ padding: var(--spacing-sm) var(--spacing-md);
+ border-radius: var(--radius-md);
+ font-weight: var(--font-weight-bold);
+ font-size: var(--font-size-caption);
+ border: 1px solid var(--color-border);
+ background: transparent;
+ color: var(--color-text-secondary);
+ cursor: pointer;
+ font-family: inherit;
+ min-height: 44px;
+ transition: color var(--transition-base), border-color var(--transition-base), background var(--transition-base);
+}
+
+.btn-ghost:hover {
+ color: var(--color-text-primary);
+ border-color: var(--color-border-strong);
+ background: var(--color-surface-elevated);
+}
+
+.btn-primary {
+ padding: var(--spacing-sm) var(--spacing-md);
+ border-radius: var(--radius-md);
+ font-weight: var(--font-weight-bold);
+ font-size: var(--font-size-caption);
+ border: none;
+ background: linear-gradient(135deg, var(--color-primary), var(--color-secondary));
+ color: var(--color-text-inverse);
+ cursor: pointer;
+ font-family: inherit;
+ min-height: 44px;
+ transition: opacity var(--transition-base), transform var(--transition-fast);
+}
+
+.btn-primary:hover {
+ opacity: 0.92;
+}
+
+.btn-primary:active {
+ transform: scale(0.98);
+}
+
+.btn-primary:disabled {
+ opacity: 0.5;
+ cursor: not-allowed;
+}
+
+.btn-primary.is-hidden,
+.apply-all-btn.is-hidden,
+.is-hidden {
+ display: none;
+}
+
+/* ── Error highlights ── */
+.spelling-error,
+.grammar-error,
+.punctuation-suggestion {
+ cursor: pointer;
+ padding: 0 2px;
+ border-radius: 2px;
+ transition: background var(--transition-fast), filter var(--transition-fast);
+}
+
+.spelling-error {
+ background: var(--highlight-spelling-bg);
+ border-bottom: 2px solid var(--highlight-spelling-border);
+}
+
+.grammar-error {
+ background: var(--highlight-grammar-bg);
+ border-bottom: 2px solid var(--highlight-grammar-border);
+}
+
+.punctuation-suggestion {
+ background: var(--highlight-punctuation-bg);
+ border-bottom: 2px solid var(--highlight-punctuation-border);
+}
+
+.spelling-error:hover,
+.grammar-error:hover,
+.punctuation-suggestion:hover,
+.highlight-active {
+ filter: brightness(1.08);
+}
+
+/* ── Suggestion popover ── */
+.suggestion-popover {
+ position: fixed;
+ z-index: 1000;
+ background: var(--color-surface-elevated);
+ border: 1px solid var(--color-border-strong);
+ border-radius: var(--radius-card);
+ padding: var(--spacing-md);
+ box-shadow: var(--shadow-popover);
+ max-width: 320px;
+ display: none;
+ opacity: 0;
+ transform: translateY(4px);
+ transition: opacity var(--transition-base), transform var(--transition-base);
+}
+
+.suggestion-popover.show {
+ display: block;
+ opacity: 1;
+ transform: translateY(0);
+}
+
+.popover-type {
+ font-size: var(--font-size-label);
+ font-weight: var(--font-weight-bold);
+ margin-bottom: var(--spacing-sm);
+ display: flex;
+ align-items: center;
+ gap: var(--spacing-sm);
+}
+
+.popover-type--spelling { color: var(--color-error); }
+.popover-type--grammar { color: var(--color-warning); }
+.popover-type--punctuation { color: var(--color-success); }
+
+.popover-correction {
+ font-size: var(--font-size-h3);
+ font-weight: var(--font-weight-bold);
+ margin-bottom: var(--spacing-sm);
+ text-align: center;
+ color: var(--color-text-primary);
+}
+
+.popover-apply {
+ width: 100%;
+ padding: var(--spacing-sm);
+ border: none;
+ border-radius: var(--radius-md);
+ background: linear-gradient(135deg, var(--color-primary), var(--color-secondary));
+ color: var(--color-text-inverse);
+ font-weight: var(--font-weight-bold);
+ cursor: pointer;
+ font-family: inherit;
+ min-height: 44px;
+ transition: opacity var(--transition-base);
+}
+
+.popover-apply:hover {
+ opacity: 0.92;
+}
+
+.popover-hint {
+ font-size: var(--font-size-label);
+ color: var(--color-text-muted);
+ text-align: center;
+ margin-top: var(--spacing-sm);
+}
+
+/* ── Suggestion cards ── */
+.suggestion-card {
+ padding: var(--spacing-md);
+ border-radius: var(--radius-md);
+ border: 1px solid var(--color-border);
+ background: var(--color-surface-elevated);
+ cursor: pointer;
+ transition: border-color var(--transition-base), background var(--transition-base);
+ text-align: right;
+}
+
+.suggestion-card:hover,
+.suggestion-card.focused {
+ border-color: var(--color-primary);
+ background: var(--color-suggestion-hover-bg);
+}
+
+.suggestion-card-badge {
+ display: inline-block;
+ font-size: var(--font-size-label);
+ font-weight: var(--font-weight-bold);
+ padding: 2px 8px;
+ border-radius: 999px;
+ margin-bottom: var(--spacing-sm);
+}
+
+.badge-spelling {
+ background: var(--color-badge-spelling-bg);
+ color: var(--color-error);
+}
+
+.badge-grammar {
+ background: var(--color-badge-grammar-bg);
+ color: var(--color-warning);
+}
+
+.badge-punctuation {
+ background: var(--color-badge-punctuation-bg);
+ color: var(--color-success);
+}
+
+.suggestion-card-change {
+ font-size: var(--font-size-caption);
+ margin-bottom: 4px;
+ color: var(--color-text-primary);
+}
+
+.suggestion-card-original {
+ text-decoration: line-through;
+ color: var(--color-text-muted);
+}
+
+.suggestion-card-arrow {
+ margin: 0 var(--spacing-sm);
+ color: var(--color-text-muted);
+}
+
+.suggestion-card-fix {
+ color: var(--color-success);
+ font-weight: var(--font-weight-semibold);
+}
+
+.suggestion-card-apply {
+ float: left;
+ padding: 4px 10px;
+ border: none;
+ border-radius: var(--radius-sm);
+ background: var(--color-primary);
+ color: var(--color-text-inverse);
+ font-size: var(--font-size-label);
+ font-weight: var(--font-weight-bold);
+ cursor: pointer;
+ font-family: inherit;
+ min-height: 32px;
+ transition: opacity var(--transition-base);
+}
+
+.suggestion-card-apply:hover {
+ opacity: 0.9;
+}
+
+/* ── Score ring ── */
+.score-ring-wrap {
+ position: relative;
+ width: 140px;
+ height: 140px;
+ margin: 0 auto var(--spacing-md);
+}
+
+.score-circle {
+ transition: stroke-dashoffset 0.6s ease-in-out;
+}
+
+.score-value {
+ position: absolute;
+ inset: 0;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ font-size: var(--font-size-h1);
+ font-weight: var(--font-weight-bold);
+ color: var(--color-text-primary);
+}
+
+/* ── Empty state ── */
+.empty-state {
+ text-align: center;
+ padding: var(--spacing-xl) var(--spacing-md);
+ color: var(--color-text-muted);
+}
+
+.empty-state-icon {
+ width: 64px;
+ height: 64px;
+ margin: 0 auto var(--spacing-md);
+ opacity: 0.35;
+ color: var(--color-text-muted);
+}
+
+/* ── Sidebar ── */
+.sidebar-panel {
+ display: flex;
+ flex-direction: column;
+ gap: var(--spacing-md);
+}
+
+.sidebar-card {
+ padding: var(--spacing-lg);
+}
+
+.suggestions-scroll {
+ max-height: 420px;
+ overflow-y: auto;
+ display: flex;
+ flex-direction: column;
+ gap: var(--spacing-sm);
+}
+
+.apply-all-btn {
+ width: 100%;
+ margin-top: var(--spacing-sm);
+}
+
+/* ── Summarize panel ── */
+.summarize-panel {
+ padding: var(--spacing-lg);
+}
+
+.summary-card {
+ padding: var(--spacing-lg);
+ border-radius: var(--radius-card);
+ background: var(--color-surface);
+ border: 1px solid var(--color-summary-accent-border);
+ box-shadow: var(--shadow-card);
+}
+
+.summary-card__title {
+ color: var(--color-secondary);
+}
+
+.summary-loading {
+ text-align: center;
+ padding: var(--spacing-lg);
+}
+
+.summary-loading__spinner {
+ display: inline-block;
+ width: 40px;
+ height: 40px;
+ border: 4px solid var(--color-summary-accent-border);
+ border-top-color: var(--color-secondary);
+ border-radius: 50%;
+ animation: spin 1s linear infinite;
+}
+
+.summary-error {
+ color: var(--color-error);
+ margin-bottom: var(--spacing-sm);
+}
+
+.summary-loading__text {
+ margin-top: var(--spacing-md);
+ color: var(--color-text-secondary);
+}
+
+.summary-preview {
+ max-height: 0;
+ overflow: hidden;
+ transition: max-height var(--transition-slow);
+}
+
+.summary-preview.show {
+ max-height: 500px;
+}
+
+/* ── Mobile bottom sheet ── */
+.mobile-sheet-trigger {
+ display: none;
+ width: 100%;
+ padding: var(--spacing-md);
+ border: 1px solid var(--color-border);
+ border-radius: var(--radius-md);
+ background: var(--color-surface);
+ color: var(--color-text-primary);
+ font-weight: var(--font-weight-bold);
+ cursor: pointer;
+ font-family: inherit;
+ min-height: 44px;
+ margin-top: var(--spacing-md);
+ transition: background var(--transition-base);
+}
+
+.mobile-sheet-trigger:hover {
+ background: var(--color-surface-elevated);
+}
+
+@media (max-width: 1023px) {
+ .mobile-sheet-trigger {
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+ }
+
+ .sidebar-desktop {
+ display: none;
+ }
+}
+
+.bottom-sheet {
+ position: fixed;
+ inset: 0;
+ z-index: 90;
+ pointer-events: none;
+ visibility: hidden;
+}
+
+.bottom-sheet.open {
+ pointer-events: auto;
+ visibility: visible;
+}
+
+.bottom-sheet-backdrop {
+ position: absolute;
+ inset: 0;
+ background: var(--color-overlay);
+ opacity: 0;
+ transition: opacity var(--transition-slow);
+}
+
+.bottom-sheet.open .bottom-sheet-backdrop {
+ opacity: 1;
+}
+
+.bottom-sheet-panel {
+ position: absolute;
+ bottom: 0;
+ left: 0;
+ right: 0;
+ max-height: 70vh;
+ background: var(--color-surface);
+ border-top: 1px solid var(--color-border-strong);
+ border-radius: var(--radius-card) var(--radius-card) 0 0;
+ padding: var(--spacing-lg);
+ transform: translateY(100%);
+ transition: transform var(--transition-slow);
+ overflow-y: auto;
+ box-shadow: var(--shadow-popover);
+}
+
+.bottom-sheet.open .bottom-sheet-panel {
+ transform: translateY(0);
+}
+
+.bottom-sheet-handle {
+ width: 40px;
+ height: 4px;
+ background: var(--color-border-strong);
+ border-radius: 2px;
+ margin: 0 auto var(--spacing-md);
+}
+
+/* ── Analysis loading ── */
+.analyzing-indicator {
+ display: none;
+ align-items: center;
+ gap: var(--spacing-sm);
+ font-size: var(--font-size-caption);
+ color: var(--color-text-secondary);
+}
+
+.analyzing-indicator.active {
+ display: flex;
+ animation: pulse-subtle 1.2s ease infinite;
+}
+
+.analyzing-spinner {
+ width: 14px;
+ height: 14px;
+ border: 2px solid var(--color-border);
+ border-top-color: var(--color-accent);
+ border-radius: 50%;
+ animation: spin 0.8s linear infinite;
+}
+
+/* ── Range input ── */
+input[type="range"] {
+ -webkit-appearance: none;
+ appearance: none;
+ background: transparent;
+ cursor: pointer;
+ flex: 1;
+}
+
+input[type="range"]::-webkit-slider-track {
+ background: var(--color-border-strong);
+ height: 6px;
+ border-radius: 3px;
+}
+
+input[type="range"]::-webkit-slider-thumb {
+ -webkit-appearance: none;
+ width: 18px;
+ height: 18px;
+ border-radius: 50%;
+ background: var(--color-primary);
+ margin-top: -6px;
+}
+
+/* ── Editor layout ── */
+.editor-layout {
+ display: grid;
+ gap: var(--spacing-md);
+ padding: var(--spacing-md);
+}
+
+@media (min-width: 1024px) {
+ .editor-layout {
+ grid-template-columns: 1fr 340px;
+ padding: var(--spacing-lg);
+ }
+}
+
+@media (min-width: 1280px) {
+ .editor-layout {
+ grid-template-columns: 1fr 380px;
+ }
+}
+
+/* ── Marketing token utilities ── */
+.marketing-card {
+ background: var(--color-surface);
+ border: 1px solid var(--color-border);
+}
+
+.marketing-divider {
+ background: var(--color-border-strong);
+}
+
+.badge-pill {
+ background: var(--color-primary-subtle-bg);
+ color: var(--color-primary);
+ border: 1px solid var(--color-primary-subtle-border);
+}
+
+.check-icon { color: var(--color-success); }
+.muted-icon { color: var(--color-text-muted); }
+
+.demo-editor-surface {
+ background: var(--color-editor);
+ color: var(--color-text-primary);
+}
+
+.demo-callout--spelling {
+ background: var(--highlight-spelling-bg);
+ border-right: 4px solid var(--highlight-spelling-border);
+}
+
+.demo-callout--grammar {
+ background: var(--highlight-grammar-bg);
+ border-right: 4px solid var(--highlight-grammar-border);
+}
+
+.demo-callout--punctuation {
+ background: var(--highlight-punctuation-bg);
+ border-right: 4px solid var(--highlight-punctuation-border);
+}
+
+.demo-callout--summarize {
+ background: var(--color-summary-accent-bg);
+ border-right: 4px solid var(--color-secondary);
+}
+
+.footer-bar {
+ background: var(--color-bg);
+ border-color: var(--color-border);
+}
+
+/* ── Document management (Phase 4) ── */
+.doc-toolbar-actions {
+ display: flex;
+ align-items: center;
+ gap: var(--spacing-sm);
+}
+
+.doc-dropdown {
+ position: relative;
+}
+
+.doc-dropdown__trigger {
+ min-height: 44px;
+}
+
+.doc-dropdown__menu {
+ position: absolute;
+ top: calc(100% + 6px);
+ left: 0;
+ min-width: 200px;
+ background: var(--color-surface-elevated);
+ border: 1px solid var(--color-border-strong);
+ border-radius: var(--radius-md);
+ box-shadow: var(--shadow-popover);
+ padding: var(--spacing-sm);
+ display: none;
+ z-index: 200;
+}
+
+.doc-dropdown__menu.is-open {
+ display: block;
+}
+
+.doc-dropdown__item {
+ display: block;
+ width: 100%;
+ text-align: right;
+ padding: var(--spacing-sm) var(--spacing-md);
+ border: none;
+ border-radius: var(--radius-sm);
+ background: transparent;
+ color: var(--color-text-primary);
+ font-family: inherit;
+ font-size: var(--font-size-caption);
+ font-weight: var(--font-weight-semibold);
+ cursor: pointer;
+ min-height: 44px;
+ transition: background var(--transition-base);
+}
+
+.doc-dropdown__item:hover:not(:disabled) {
+ background: var(--color-suggestion-hover-bg);
+}
+
+.doc-dropdown__item:disabled {
+ opacity: 0.45;
+ cursor: not-allowed;
+}
+
+.analysis-limit-banner {
+ margin: 0 var(--spacing-md) var(--spacing-sm);
+ padding: var(--spacing-sm) var(--spacing-md);
+ border-radius: var(--radius-md);
+ background: var(--color-primary-subtle-bg);
+ border: 1px solid var(--color-primary-subtle-border);
+ color: var(--color-text-secondary);
+ font-size: var(--font-size-caption);
+ text-align: right;
+}
+
+.doc-toast {
+ position: fixed;
+ bottom: var(--spacing-lg);
+ left: 50%;
+ transform: translateX(-50%) translateY(20px);
+ padding: var(--spacing-sm) var(--spacing-lg);
+ border-radius: var(--radius-md);
+ background: var(--color-surface-elevated);
+ border: 1px solid var(--color-border-strong);
+ color: var(--color-text-primary);
+ box-shadow: var(--shadow-popover);
+ font-size: var(--font-size-caption);
+ font-weight: var(--font-weight-semibold);
+ z-index: 300;
+ opacity: 0;
+ pointer-events: none;
+ transition: opacity var(--transition-base), transform var(--transition-base);
+}
+
+.doc-toast.is-visible {
+ opacity: 1;
+ transform: translateX(-50%) translateY(0);
+}
+
+.doc-toast--success {
+ border-color: var(--color-success);
+}
+
+.doc-toast--error {
+ border-color: var(--color-error);
+}
+
+/* PDF export styles are applied inline in buildPdfHtmlString for clone reliability */
+
+.doc-mobile-actions {
+ display: flex;
+ gap: var(--spacing-sm);
+ width: 100%;
+ margin-top: var(--spacing-sm);
+}
+
+@media (max-width: 639px) {
+ .doc-toolbar-actions--desktop {
+ display: none !important;
+ }
+}
+
+@media (min-width: 640px) {
+ .doc-mobile-actions {
+ display: none;
+ }
+}
+
+/* ── Authentication (Phase 5) ── */
+.auth-offline-banner {
+ position: fixed;
+ top: 64px;
+ left: 0;
+ right: 0;
+ z-index: 45;
+ padding: var(--spacing-sm) var(--spacing-md);
+ text-align: center;
+ font-size: var(--font-size-caption);
+ background: var(--color-primary-subtle-bg);
+ border-bottom: 1px solid var(--color-primary-subtle-border);
+ color: var(--color-text-secondary);
+}
+
+.auth-gate {
+ position: fixed;
+ inset: 0;
+ z-index: 400;
+ display: none;
+ align-items: center;
+ justify-content: center;
+}
+
+.auth-gate.is-open {
+ display: flex;
+}
+
+.auth-gate-backdrop {
+ position: absolute;
+ inset: 0;
+ background: rgba(0, 0, 0, 0.55);
+ backdrop-filter: blur(4px);
+}
+
+.auth-gate-panel {
+ position: relative;
+ z-index: 1;
+ background: var(--color-surface-elevated);
+ border: 1px solid var(--color-border-strong);
+ border-radius: var(--radius-lg);
+ box-shadow: var(--shadow-popover);
+ padding: var(--spacing-xl);
+ max-width: 420px;
+ width: calc(100% - 2rem);
+ text-align: center;
+}
+
+.auth-gate-panel--mobile {
+ display: none;
+}
+
+.auth-gate-title {
+ font-size: var(--font-size-h2);
+ font-weight: var(--font-weight-bold);
+ margin: 0 0 var(--spacing-sm);
+ color: var(--color-text-primary);
+}
+
+.auth-gate-subtitle {
+ font-size: var(--font-size-body);
+ color: var(--color-text-secondary);
+ margin: 0 0 var(--spacing-lg);
+}
+
+.auth-gate-actions {
+ display: flex;
+ flex-direction: column;
+ gap: var(--spacing-sm);
+}
+
+.auth-gate-btn {
+ width: 100%;
+ min-height: 48px;
+ justify-content: center;
+}
+
+.auth-gate-btn--google {
+ display: inline-flex;
+ align-items: center;
+ gap: var(--spacing-sm);
+}
+
+.auth-menu-wrap.is-hidden {
+ display: none;
+}
+
+.auth-dropdown {
+ position: relative;
+}
+
+.auth-menu-trigger {
+ display: inline-flex;
+ align-items: center;
+ gap: var(--spacing-sm);
+ min-height: 44px;
+ padding: 0 var(--spacing-sm);
+ border: 1px solid var(--color-border);
+ border-radius: var(--radius-md);
+ background: var(--color-surface);
+ color: var(--color-text-primary);
+ cursor: pointer;
+ font-family: inherit;
+ font-size: var(--font-size-caption);
+}
+
+.auth-menu-trigger:hover {
+ background: var(--color-surface-elevated);
+}
+
+.auth-avatar {
+ display: inline-flex;
+ align-items: center;
+ justify-content: center;
+ width: 32px;
+ height: 32px;
+ border-radius: 50%;
+ background: var(--color-primary-subtle-bg);
+ color: var(--color-primary);
+ font-weight: var(--font-weight-bold);
+ overflow: hidden;
+}
+
+.auth-avatar-img {
+ width: 100%;
+ height: 100%;
+ object-fit: cover;
+}
+
+.auth-display-name {
+ max-width: 120px;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+}
+
+.auth-menu-chevron {
+ opacity: 0.6;
+ font-size: 0.75rem;
+}
+
+.auth-account-menu {
+ position: absolute;
+ top: calc(100% + 6px);
+ left: 0;
+ min-width: 220px;
+ background: var(--color-surface-elevated);
+ border: 1px solid var(--color-border-strong);
+ border-radius: var(--radius-md);
+ box-shadow: var(--shadow-popover);
+ padding: var(--spacing-sm);
+ display: none;
+ z-index: 200;
+ text-align: right;
+}
+
+.auth-account-menu.is-open {
+ display: block;
+}
+
+.auth-account-menu__header {
+ padding: var(--spacing-sm) var(--spacing-md);
+ border-bottom: 1px solid var(--color-border);
+ margin-bottom: var(--spacing-sm);
+}
+
+.auth-provider-label {
+ font-size: var(--font-size-caption);
+ color: var(--color-text-muted);
+}
+
+.auth-account-menu__item {
+ display: block;
+ width: 100%;
+ text-align: right;
+ padding: var(--spacing-sm) var(--spacing-md);
+ border: none;
+ border-radius: var(--radius-sm);
+ background: transparent;
+ color: var(--color-text-primary);
+ font-family: inherit;
+ font-size: var(--font-size-caption);
+ cursor: pointer;
+ min-height: 44px;
+}
+
+.auth-account-menu__item:hover {
+ background: var(--color-suggestion-hover-bg);
+}
+
+.auth-account-menu__item--danger {
+ color: var(--color-error);
+}
+
+.auth-account-menu__item.is-hidden {
+ display: none;
+}
+
+.auth-drawer-section {
+ margin-top: var(--spacing-md);
+ padding-top: var(--spacing-md);
+ border-top: 1px solid var(--color-border);
+}
+
+.auth-drawer-label {
+ font-size: var(--font-size-caption);
+ color: var(--color-text-muted);
+ margin: 0 0 var(--spacing-xs);
+}
+
+.auth-drawer-name {
+ font-weight: var(--font-weight-semibold);
+ margin: 0 0 2px;
+}
+
+.auth-drawer-provider {
+ font-size: var(--font-size-caption);
+ color: var(--color-text-secondary);
+ margin: 0 0 var(--spacing-sm);
+}
+
+.auth-drawer-action.is-hidden {
+ display: none;
+}
+
+@media (max-width: 639px) {
+ .auth-gate-panel--desktop {
+ display: none;
+ }
+
+ .auth-gate-panel--mobile {
+ display: block;
+ position: fixed;
+ bottom: 0;
+ left: 0;
+ right: 0;
+ max-width: none;
+ width: 100%;
+ border-radius: var(--radius-lg) var(--radius-lg) 0 0;
+ padding-bottom: calc(var(--spacing-xl) + env(safe-area-inset-bottom, 0));
+ }
+
+ .auth-display-name {
+ display: none;
+ }
+}
diff --git a/src/css/tokens.css b/src/css/tokens.css
new file mode 100644
index 0000000000000000000000000000000000000000..73ee41d1a28f6bce102e5a0fda0a0abec7c18a80
--- /dev/null
+++ b/src/css/tokens.css
@@ -0,0 +1,169 @@
+/**
+ * Bayan Design System — Theme Tokens
+ * Comfortable palettes for long Arabic reading (Notion / Grammarly inspired)
+ */
+
+/* ── Shared structural tokens ── */
+:root {
+ --font-family-primary: 'Cairo', 'Tajawal', 'Noto Sans Arabic', sans-serif;
+
+ --font-size-display: 3rem;
+ --font-size-h1: 2.25rem;
+ --font-size-h2: 1.875rem;
+ --font-size-h3: 1.25rem;
+ --font-size-body: 1rem;
+ --font-size-editor: 1.125rem;
+ --font-size-caption: 0.875rem;
+ --font-size-label: 0.75rem;
+
+ --font-weight-regular: 400;
+ --font-weight-medium: 500;
+ --font-weight-semibold: 600;
+ --font-weight-bold: 700;
+
+ --line-height-editor: 1.9;
+ --line-height-body: 1.75;
+ --letter-spacing-arabic: 0.015em;
+
+ --spacing-xs: 0.25rem;
+ --spacing-sm: 0.5rem;
+ --spacing-md: 1rem;
+ --spacing-lg: 1.5rem;
+ --spacing-xl: 2rem;
+
+ --radius-sm: 0.5rem;
+ --radius-md: 0.75rem;
+ --radius-lg: 1rem;
+ --radius-card: 1rem;
+
+ --transition-fast: 0.15s ease;
+ --transition-base: 0.2s ease;
+ --transition-slow: 0.3s ease;
+}
+
+/* ── Dark Theme — soft charcoal, easy on eyes ── */
+:root,
+[data-theme="dark"] {
+ color-scheme: dark;
+
+ --color-bg: #12141A;
+ --color-surface: #1A1D26;
+ --color-surface-elevated: #242833;
+ --color-editor: #1E2229;
+
+ --color-primary: #6BA3E0;
+ --color-secondary: #A594E8;
+ --color-accent: #6BBECF;
+
+ --color-success: #6BC98A;
+ --color-warning: #E4B35A;
+ --color-error: #E88A8A;
+
+ --color-text-primary: #ECEEF2;
+ --color-text-secondary: #B4BBC6;
+ --color-text-muted: #8A939F;
+ --color-text-inverse: #F4F5F7;
+ --color-placeholder: #9AA3AE;
+
+ --color-border: rgba(236, 238, 242, 0.09);
+ --color-border-strong: rgba(236, 238, 242, 0.16);
+
+ --focus-ring: rgba(107, 163, 224, 0.42);
+
+ --shadow-card: 0 8px 24px rgba(0, 0, 0, 0.28);
+ --shadow-popover: 0 16px 40px rgba(0, 0, 0, 0.38);
+ --shadow-editor: 0 2px 16px rgba(0, 0, 0, 0.2);
+
+ --color-overlay: rgba(8, 10, 14, 0.62);
+
+ --highlight-spelling-bg: rgba(232, 138, 138, 0.16);
+ --highlight-spelling-border: #E88A8A;
+ --highlight-grammar-bg: rgba(228, 179, 90, 0.16);
+ --highlight-grammar-border: #E4B35A;
+ --highlight-punctuation-bg: rgba(107, 201, 138, 0.14);
+ --highlight-punctuation-border: #6BC98A;
+
+ --color-nav-bg: rgba(18, 20, 26, 0.94);
+ --color-badge-spelling-bg: rgba(232, 138, 138, 0.18);
+ --color-badge-grammar-bg: rgba(228, 179, 90, 0.18);
+ --color-badge-punctuation-bg: rgba(107, 201, 138, 0.16);
+ --color-summary-accent-bg: rgba(165, 148, 232, 0.14);
+ --color-summary-accent-border: rgba(165, 148, 232, 0.38);
+ --color-suggestion-hover-bg: rgba(107, 163, 224, 0.12);
+ --color-primary-subtle-bg: rgba(107, 163, 224, 0.14);
+ --color-primary-subtle-border: rgba(107, 163, 224, 0.32);
+
+ --primary-color: var(--color-primary);
+ --secondary-color: var(--color-secondary);
+ --text-color: var(--color-text-primary);
+ --text-secondary: var(--color-text-secondary);
+ --surface-color: var(--color-surface);
+ --background-color: var(--color-bg);
+ --success-color: var(--color-success);
+ --warning-color: var(--color-warning);
+ --error-color: var(--color-error);
+ --nav-bg: var(--color-nav-bg);
+}
+
+/* ── Light Theme — warm paper, strong readable text ── */
+[data-theme="light"] {
+ color-scheme: light;
+
+ --color-bg: #F3F1EC;
+ --color-surface: #FAF9F6;
+ --color-surface-elevated: #EFEBE4;
+ --color-editor: #FFFDF8;
+
+ --color-primary: #2B6CB8;
+ --color-secondary: #6B57A8;
+ --color-accent: #2A8F9E;
+
+ --color-success: #2F8554;
+ --color-warning: #B7791F;
+ --color-error: #C53030;
+
+ --color-text-primary: #1A1D21;
+ --color-text-secondary: #3A424E;
+ --color-text-muted: #5A6472;
+ --color-text-inverse: #FAFAF8;
+ --color-placeholder: #6B7580;
+
+ --color-border: rgba(26, 29, 33, 0.1);
+ --color-border-strong: rgba(26, 29, 33, 0.18);
+
+ --focus-ring: rgba(43, 108, 184, 0.32);
+
+ --shadow-card: 0 6px 20px rgba(26, 29, 33, 0.07);
+ --shadow-popover: 0 14px 36px rgba(26, 29, 33, 0.11);
+ --shadow-editor: 0 1px 8px rgba(26, 29, 33, 0.05);
+
+ --color-overlay: rgba(26, 29, 33, 0.38);
+
+ --highlight-spelling-bg: #FDECEC;
+ --highlight-spelling-border: #C53030;
+ --highlight-grammar-bg: #FEF6E4;
+ --highlight-grammar-border: #B7791F;
+ --highlight-punctuation-bg: #EAF6EE;
+ --highlight-punctuation-border: #2F8554;
+
+ --color-nav-bg: rgba(250, 249, 246, 0.94);
+ --color-badge-spelling-bg: rgba(197, 48, 48, 0.1);
+ --color-badge-grammar-bg: rgba(183, 121, 31, 0.12);
+ --color-badge-punctuation-bg: rgba(47, 133, 84, 0.1);
+ --color-summary-accent-bg: rgba(107, 87, 168, 0.08);
+ --color-summary-accent-border: rgba(107, 87, 168, 0.22);
+ --color-suggestion-hover-bg: rgba(43, 108, 184, 0.07);
+ --color-primary-subtle-bg: rgba(43, 108, 184, 0.09);
+ --color-primary-subtle-border: rgba(43, 108, 184, 0.22);
+
+ --primary-color: var(--color-primary);
+ --secondary-color: var(--color-secondary);
+ --text-color: var(--color-text-primary);
+ --text-secondary: var(--color-text-secondary);
+ --surface-color: var(--color-surface);
+ --background-color: var(--color-bg);
+ --success-color: var(--color-success);
+ --warning-color: var(--color-warning);
+ --error-color: var(--color-error);
+ --nav-bg: var(--color-nav-bg);
+}
diff --git a/src/index.html b/src/index.html
index 0599d6c74bdceff0e21f20d36a708f8a41f10b56..db13839e35d39ece232d91ebd4261766d5f0d5d8 100644
--- a/src/index.html
+++ b/src/index.html
@@ -3,250 +3,128 @@
+
+
بيان - مساعد الكتابة العربية الذكي
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-