Dedupe repeated equation blocks in region OCR output

app.py

@@ -85,8 +85,19 @@ TASK_PROMPTS = {
 
 def extract_grounding_references(text):
     refs = []
+    seen = set()
     for entry in _extract_grounding_entries(text):
         coord_text = repr(entry["coords"])
+        key = (
+            entry["label"].strip().lower(),
+            tuple(
+                (round(c[0], 1), round(c[1], 1), round(c[2], 1), round(c[3], 1))
+                for c in entry["coords"]
+            ),
+        )
+        if key in seen:
+            continue
+        seen.add(key)
         raw = f'<|ref|>{entry["label"]}<|/ref|><|det|>{coord_text}<|/det|>'
         refs.append((raw, entry["label"], coord_text))
     return refs
@@ -249,6 +260,7 @@ def clean_output(text, include_images=False):
             text = re.sub(rf'(?m)^[^\n]*{re.escape(match[0])}[^\n]*\n?', '', text)
 
     text = _strip_malformed_grounding(text)
+    text = _dedupe_repeated_math_blocks(text)
     return text.strip()
 
 def _strip_malformed_grounding(text: str) -> str:
@@ -268,6 +280,46 @@ def _strip_malformed_grounding(text: str) -> str:
     text = re.sub(r'<\|/?det\|>', '', text)
     return text
 
+def _equation_text_key(text: str) -> str:
+    if not text:
+        return ""
+    key = text.strip()
+    key = re.sub(r'\\\[(.+?)\\\]', r'\1', key, flags=re.DOTALL)
+    key = re.sub(r'\\\((.+?)\\\)', r'\1', key, flags=re.DOTALL)
+    key = re.sub(r'\$\$(.+?)\$\$', r'\1', key, flags=re.DOTALL)
+    key = re.sub(r'\^\{([A-Za-z0-9])\}', r'^\1', key)
+    key = re.sub(r'_\{([A-Za-z0-9])\}', r'_\1', key)
+    key = re.sub(r'\s+', '', key)
+    return key.lower()
+
+def _dedupe_repeated_math_blocks(text: str) -> str:
+    if not text:
+        return ""
+
+    pattern = re.compile(r'\\\[(.+?)\\\]|\\\((.+?)\\\)|\$\$(.+?)\$\$', re.DOTALL)
+    seen = set()
+    out = []
+    last = 0
+    removed_any = False
+
+    for m in pattern.finditer(text):
+        out.append(text[last:m.start()])
+        expr = m.group(1) or m.group(2) or m.group(3) or ""
+        key = _equation_text_key(expr)
+        if key and key in seen:
+            removed_any = True
+        else:
+            if key:
+                seen.add(key)
+            out.append(m.group(0))
+        last = m.end()
+    out.append(text[last:])
+
+    merged = ''.join(out)
+    if removed_any:
+        merged = re.sub(r'\n{3,}', '\n\n', merged)
+    return merged
+
 PREVIEW_CSS = """
 <style>
 .math-preview {
@@ -869,6 +921,7 @@ def _process_equation_lines_separately(image, infer_crop_mode=None):
     raw_parts = [f"## Detection\n\n{detect_raw}".strip()]
     refs = []
     crops = []
+    seen_line_keys = set()
 
     for i, box in enumerate(boxes, 1):
         x1, y1, x2, y2 = _norm_box_to_pixels(box, img_w, img_h, pad_ratio=0.01)
@@ -877,6 +930,11 @@ def _process_equation_lines_separately(image, infer_crop_mode=None):
         line_clean = clean_output(line_raw, False).strip()
         if not line_clean:
             continue
+        line_key = _equation_text_key(line_clean)
+        if line_key and line_key in seen_line_keys:
+            continue
+        if line_key:
+            seen_line_keys.add(line_key)
         line_label = f"Eq {i}"
         line_markdown = line_clean
         if "$$" not in line_markdown and "\\[" not in line_markdown and "\\(" not in line_markdown:

tests/test_clean_output.py

@@ -10,6 +10,8 @@ def _load_clean_output():
     module = ast.parse(source, filename=str(app_path))
 
     wanted = {
+        "_equation_text_key",
+        "_dedupe_repeated_math_blocks",
         "_strip_malformed_grounding",
         "clean_output",
     }
@@ -49,6 +51,16 @@ class CleanOutputTests(unittest.TestCase):
         self.assertNotIn("<|ref|>", cleaned)
         self.assertNotIn("<|det|>", cleaned)
 
+    def test_dedupes_equivalent_math_blocks(self):
+        clean_output = _load_clean_output()
+        raw = (
+            "\\[ \\frac{(18x-27-7)}{(2x-3)^2}=\\frac{18x-34}{(2x-3)^2} \\]\n"
+            "\\[ \\frac{(18x-27-7)}{(2x-3)^{2}}=\\frac{18x-34}{(2x-3)^{2}} \\]\n"
+        )
+
+        cleaned = clean_output(raw, include_images=True)
+        self.assertEqual(1, cleaned.count("\\["))
+
 
 if __name__ == "__main__":
     unittest.main()