Spaces:
Sleeping
Sleeping
File size: 4,793 Bytes
d473af5 81a53ea d473af5 81a53ea d473af5 81a53ea d473af5 81a53ea d473af5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 | import os
import re
from pathlib import Path
from typing import List, Tuple
import fitz # pymupdf
from pdf2image import convert_from_path
import pytesseract
from PIL import ImageOps, ImageEnhance
OCR_LANG = "eng+ara"
OCR_DPI = 180
NATIVE_MIN_CHARS_PER_PAGE = 60 # if native extracted text < this => OCR that page
_SENT_BOUNDARY_RE = re.compile(r"(?<=[\.\!\?\u061F\u06D4\u061B…])\s+") # . ! ? ؟ ۔ ؛ …
def normalize_text(text: str) -> str:
"""Normalizes text by removing excessive whitespace and fixing newlines."""
text = text.replace("\r\n", "\n").replace("\r", "\n")
text = re.sub(r"[ \t]+", " ", text)
text = re.sub(r"\n{3,}", "\n\n", text)
return text.strip()
def ocr_image_pil(img):
"""Applies light preprocessing to improve OCR accuracy."""
img = img.convert("RGB")
img = ImageOps.grayscale(img)
img = ImageEnhance.Contrast(img).enhance(1.6)
return img
def ocr_pdf_page(pdf_path: str, page_number_1based: int, dpi: int = OCR_DPI, lang: str = OCR_LANG) -> str:
"""OCRs a single PDF page."""
images = convert_from_path(
str(pdf_path),
dpi=dpi,
first_page=page_number_1based,
last_page=page_number_1based,
fmt="png",
thread_count=2,
)
if not images:
return ""
img = images[0]
img = ocr_image_pil(img)
return pytesseract.image_to_string(img, lang=lang)
def pdf_to_text_smart(pdf_path: str, native_min_chars_per_page: int = NATIVE_MIN_CHARS_PER_PAGE) -> str:
"""Extracts text from PDF, falling back to OCR for scanned pages.
Optimized to avoid OCR on native PDFs with sparse pages (like title pages)."""
doc = fitz.open(str(pdf_path))
parts = []
# Quick check: is this likely a native PDF?
# Sample up to 10 pages to see if any has a good amount of native text.
is_native_pdf = False
sample_pages = min(10, doc.page_count)
for i in range(sample_pages):
page = doc.load_page(i)
native = (page.get_text("text") or "").strip()
if len(re.sub(r"\s+", "", native)) > 200:
is_native_pdf = True
break
for i in range(doc.page_count):
page = doc.load_page(i)
native = (page.get_text("text") or "").strip()
native_compact_len = len(re.sub(r"\s+", "", native))
if native_compact_len >= native_min_chars_per_page or is_native_pdf:
# If we know it's a native PDF, even sparse pages (like titles/blank pages) don't need OCR
parts.append(native)
else:
# Only OCR if it's not a known native PDF and native text is sparse (could be a scanned page)
ocr = ocr_pdf_page(pdf_path, page_number_1based=i+1)
parts.append(ocr)
doc.close()
return normalize_text("\n\n".join(parts))
def extract_text_from_file(file_path: str) -> str:
"""Extracts text from a .txt or .pdf file."""
path = Path(file_path)
suf = path.suffix.lower()
if suf == ".txt":
raw = path.read_text(encoding="utf-8", errors="ignore")
return normalize_text(raw)
if suf == ".pdf":
return pdf_to_text_smart(str(path))
raise ValueError(f"Unsupported file type '{suf}'. Please upload .pdf or .txt only.")
def split_into_chapters(text: str) -> List[Tuple[str, str]]:
"""
Best effort chapter split:
- Detect lines that look like: CHAPTER 1 / Chapter One / CHAPTER ONE etc.
- If not found, return one chapter = full text.
Returns: list of (title, body)
"""
text = normalize_text(text)
lines = text.splitlines()
chapter_re = re.compile(r"^\s*(chapter|CHAPTER)\s+([0-9]+|[IVXLC]+|[A-Za-z]+)\b.*$", re.IGNORECASE)
idxs = []
titles = []
for i, ln in enumerate(lines):
if chapter_re.match(ln.strip()):
idxs.append(i)
titles.append(ln.strip())
if len(idxs) < 2:
return [("BOOK", text)]
chapters = []
for k in range(len(idxs)):
start = idxs[k]
end = idxs[k+1] if k+1 < len(idxs) else len(lines)
title = titles[k]
body = "\n".join(lines[start:end]).strip()
chapters.append((title, body))
return chapters
def split_sentences(paragraph: str) -> List[str]:
"""Splits a paragraph into sentences."""
paragraph = paragraph.strip()
if not paragraph:
return []
if not any(ch in paragraph for ch in ".!?\u061F\u06D4\u061B…"):
ls = [ln.strip() for ln in paragraph.split("\n") if ln.strip()]
return ls if ls else [paragraph]
return [s.strip() for s in _SENT_BOUNDARY_RE.split(paragraph) if s.strip()]
def iter_paragraphs(text: str):
"""Yields paragraphs from text."""
for p in re.split(r"\n\s*\n+", text):
p = p.strip()
if p:
yield p
|