scrub_pii.py

"""Regex PII scrub for Polish web/official text.

Replaces emails, phones, PESEL/NIP/REGON/KRS, land-register (KW) numbers, electronic contact addresses and
account numbers in place so sentence structure survives. Names of public officials are left untouched —
that is intentional, not a gap. Phones map to [Telefon]; everything else
to [PII]. Bare PESEL requires checksum and date validation; explicit identifier
labels also redact damaged numbers (including common OCR O/I/l substitutions).
NIP/REGON/KRS and identity documents require explicit nearby labels. KW numbers
need a nearby KW/księga wieczysta label, or the full XX0X/00000000/0 form with a
valid check digit. Passport
variants include one-letter and diplomatic IDs. Foreign IBANs use country
lengths and MOD-97. ePUAP paths require a label; e-Doreczenia uses its AE:PL form.
Wrapped email domains, local parts hyphenated across one line break and small
extraction gaps around @/hyphens are supported. A title-case alphabetic prefix
of 5–11 letters is dropped from the redaction when the remainder is already a
complete lowercase-local email and the same passage has at least two such glues.
Explicit phone extensions and terminal suffix ranges are included; room numbers are not. Labelled numeric
PINs (including URL pin= values) map to [PII] before phone detection.
Contact/helpline headings cover consecutive descriptive phone-list entries;
unrelated lines end the list. Bounded staff/address-directory evidence also covers
formatted phone fields. Labelled full-number ranges retain shared prefixes.
Phones require a nearby contact cue, a Polish +48/0048 prefix, or explicit
international country/trunk notation such as +CC (0). With a cue, the EUR-Lex
"(32-2) 299 11 11" country-area form counts as international. Strong labels also
admit one-digit country codes and wider hyphenated area codes. Short service numbers
need strong labels; 116xxx numbers also accept nearby telephone prose. Unlabelled
domestic numbers are left for audit because table cells have the same shapes.
Flattened tables can glue labels to both neighbours ("Mödlingtel.: … 38112faks:");
such glued labels count as labels and end the preceding number.

Call after HTML-to-text, before the parquet is written.
"""
from __future__ import annotations
import datetime as dt
import re

PHONE_TAG = "[Telefon]"
PII_TAG = "[PII]"
COUNTS = ("email", "phone", "pesel", "nip", "regon", "account", "document", "krs", "electronic_address", "pin",
          "land_register")

# Mobile + geographic area codes (2-digit national prefix after trunk 0 / +48).
_PL_PREFIX = {
    "12", "13", "14", "15", "16", "17", "18", "22", "23", "24", "25", "26", "29",
    "32", "33", "34", "39", "41", "42", "43", "44", "45", "46", "47", "48",
    "50", "51", "52", "53", "54", "55", "56", "57", "58", "59",
    "60", "61", "62", "63", "65", "66", "67", "68", "69",
    "70", "71", "72", "73", "74", "75", "76", "77", "78", "79",
    "80", "81", "82", "83", "84", "85", "86", "87", "88", "89",
    "91", "94", "95",
}

_EMAIL_DOMAIN = r"[^\W_](?:[^\W_]|-[ \t]{0,3}(?=[^\W_]))*"
# PDF extraction can hyphenate a local part across one line break: "jan-\nna.k@example.com".
# The fragment is capped at the 64-character local-part limit so long tokens stay linear.
_EMAIL_START = r"\b(?:[\w.%+&-]{0,63}[^\W_]-[ \t]*\r?\n[ \t]*)?[\w.%+&-]+[ \t]{0,3}@"
_EMAIL_RE = re.compile(_EMAIL_START + r"[ \t]{0,3}(?:" + _EMAIL_DOMAIN + r"\.)+[^\W\d_]{2,}\b")
_EMAIL_WRAP_RE = re.compile(_EMAIL_START + r"[ \t]{0,3}(?:" + _EMAIL_DOMAIN
    + r"\.)+[ \t]*\r?\n[ \t]*(?:" + _EMAIL_DOMAIN + r"\.)*[^\W\d_]{2,}\b")
# Damaged contact fields can retain only the local part and @.
_EMAIL_FRAGMENT_RE = re.compile(_EMAIL_START + r"(?=[ \t]*\r?$)", re.M)
_EMAIL_LABEL_RE = re.compile(r"\be[ -]?mail[ \t]*:[ \t]*\Z", re.I)
_EDELIVERY_RE = re.compile(r"\bAE:PL-\d{5}-\d{5}-[A-Z0-9]{5}-\d{2}(?![\w-])", re.I)
_EPUAP_RE = re.compile(r"(?<![\w/])/[\w-]+/[\w-]+(?![\w/-])")
_EPUAP_LABEL_RE = re.compile(
    r"\b(?:e[ -]?puap|elektroniczna[ \t]+skrzynka[ \t]+podawcza)\b[^\n/\[\]]{0,50}\Z", re.I)
_PIN_RE = re.compile(
    r"(?P<label>\bPIN[ \t]*[:=][ \t]*)"
    r"(?P<number>[\u202a-\u202e\u2066-\u2069]*\d(?:[ \t]?\d){3,15}#?)"
    r"(?!\w|[ \t]*\d)", re.I)
# Horizontal Unicode spaces from HTML/PDF extraction; preserve paragraph breaks.
_SPACES = str.maketrans({c: " " for c in "\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u202f\u205f\u3000"})
# Country lengths checked against Apache Commons Validator's IBAN registry table:
# https://commons.apache.org/proper/commons-validator/xref/org/apache/commons/validator/routines/IBANValidator.html
# ponytail: country/length/MOD-97 only; add national BBAN rules if false positives appear.
_IBAN_LENGTHS = {
    country: length for length, countries in {
        15: "NO", 16: "BE", 18: "DK FI AX FK FO GL NL SD",
        19: "MK SI", 20: "AT BA EE KZ LT LU MN XK",
        21: "CH HR LI LV", 22: "BG BH CR DE GB IM JE GG GE IE ME RS VA",
        23: "AE GI IL IQ OM SO TL", 24: "AD CZ ES MD PK RO SA SE SK TN VG",
        25: "LY PT ST", 26: "IS TR",
        27: "BI DJ FR GF GP MQ RE PF TF YT NC BL MF PM WF GR IT MC MR SM",
        28: "AL AZ BY CY DO GT HN HU LB NI PL SV", 29: "BR EG PS QA UA",
        30: "JO KW MU YE", 31: "MT SC", 32: "LC", 33: "RU",
    }.items() for country in countries.split()
}
_IBAN_RE = re.compile(
    r"\b(?:" + "|".join(
        country + r"[ \t-]*[0-9]{2}(?:[ \t-]*[A-Z0-9]){" + str(length - 4) + "}"
        for country, length in _IBAN_LENGTHS.items()
    ) + r")\b", re.I,
)
# Optional PL, then 26 digits with short space/tab/hyphen gaps (invoice style).
_ACCOUNT_RE = re.compile(r"\b(?:PL[ \t-]*)?(?:\d[ \t-]*){25}\d\b", re.I)
_REGON14_RE = re.compile(r"\b\d{14}\b")
_PESEL_RE = re.compile(r"\b\d{11}\b")
_NIP_DASH_RE = re.compile(r"\b(?:PL[ \t]*)?\d{3}(?:[- \t]\d{3}[- \t]\d{2}[- \t]\d{2}|[- \t]\d{2}[- \t]\d{2}[- \t]\d{3})\b", re.I)
_NIP_RE = re.compile(r"\b(?:PL[ \t]*)?\d{10}\b", re.I)
_DOCUMENT_RE = re.compile(
    r"\b(?P<label>(?:dow[oó]d(?:u|em)?[ \t]+osobist(?:y|ego|ym)|"
    r"(?:nr|numer)[ \t]+dowodu|paszport(?:u|em)?(?:[ \t]+dyplomatyczn(?:y|ego|ym))?)"
    r"[ \t]*(?:(?:seria[ \t]+i[ \t]+numer|serii|seria|numer|nr)\.?[ \t]*)?"
    r"[:=-]?[ \t]*)"
    r"(?P<number>[A-Z]{3}[ \t-]?[0-9OIl]{6}|[A-Z]{2}[ \t-]?[0-9OIl]{7}|[A-Z][ \t-]?[0-9]{6,9}|[0-9]{4,12})\b", re.I,
)
_LABELLED_ID_RE = re.compile(
    r"\b(?P<label>(?P<kind>PESEL|NIP|REGON)\b"
    r"(?:[ \t]+(?:Gminy|Powiatu|Miasta|firmy)(?:[ \t]+[^\W\d_][^\W\d_-]*){0,4})?[ \t]{0,8}"
    r"(?:(?:nr\.?|numer)[ \t]{0,8})?[:=.\-]?[ \t]{0,8}(?:\r?\n[ \t]{0,8})?)"
    r"(?P<number>(?:PL[ \t]*)?(?:[0-9OIl]{9,14}|\d(?:[ \t-]?\d){8,13}))"
    r"(?!\w|[ \t-]*\d)", re.I,
)
# Land-register (KW) number: court code / number / check digit, e.g. WA1M/00123456/3.
_LAND_REGISTER_RE = re.compile(
    r"\b(?P<court>[A-Z]{2}\d[A-Z])[ \t]*[/ \t][ \t]*(?P<number>\d{1,8})[ \t]*/[ \t]*(?P<check>\d)(?![\w/])")
_LAND_REGISTER_LABEL_RE = re.compile(
    r"(?:(?-i:\bKW\b)|\bks(?:\.|i[eęą]\w*)[ \t]+wieczyst\w*)[^\n]{0,30}\Z", re.I)
# Check-digit values; court codes skip Q and V.
_LAND_REGISTER_VALUES = {c: i for i, c in enumerate("0123456789XABCDEFGHIJKLMNOPRSTUWYZ")}
_REGON9_RE = re.compile(r"\b\d{9}\b")
_ID_LABEL_END = r"[ \t]{0,8}(?:(?:nr\.?|numer)[ \t]{0,8})?[:=.\-]?[ \t]{0,8}(?:\r?\n[ \t]{0,8})?\Z"
_NIP_LABEL_RE = re.compile(r"\bNIP\b" + _ID_LABEL_END, re.I)
_REGON_LABEL_RE = re.compile(r"\bREGON\b" + _ID_LABEL_END, re.I)
_KRS_RE = re.compile(r"\b\d{6,10}(?!\w|[ \t-]*\d)")
# Abbreviation or written-out register name, optionally "pod nr/numerem".
_KRS_LABEL_RE = re.compile(
    r"(?:\bKRS\b|\bKrajow\w*[ \t]+Rejestr\w*[ \t]+S[aą]dow\w*)"
    r"(?:[ \t]{0,8},?[ \t]{0,8}pod[ \t]{1,8}(?:nr\.?|numerem))?" + _ID_LABEL_END, re.I)
# Separators are short and local: one newline *or* a few punct/spaces.
# Letters and blank lines break the match so body text is not swallowed.
_SEP = r"(?:[ \t./\-()\u2013\u2014]{0,3}|\n)"
_VANITY_PHONE_RE = re.compile(
    r"(?<!\w)\+1[ \t-]+\d{3}[ \t-]+(?:\d{3}-[A-Z]{4}|[A-Z]{7})"
    r"(?:[ \t]+\(\d{4,7}\))?(?!\w)")
# Flattened tables glue labels to both neighbours: "Mödlingtel.: +43 (0) 512 34-56712faks:".
# Glued "tel" needs its dot and glued "fax" a non-letter before it ("Hotel:", "Halifax:").
_GLUED_PHONE_LABEL = r"(?:tel\.|telefon|t[ée]l[ée]phone|t[ée]l[ée]copieur|(?<![^\W\d_])fa(?:x|ks))[ \t]*:"
# A number ends at a word boundary or where a glued contact label starts.
_PHONE_END = r"(?:(?!\w)|(?=" + _GLUED_PHONE_LABEL + r"|(?:e-?mail|t[ée]lex)[ \t]*:))"
_PHONE_RE = re.compile(
    _VANITY_PHONE_RE.pattern + r"|(?:(?<!\w)|(?<=telefon)|(?<=tel)|(?<=faks)|(?<=fax))(?:\(?\+[ \t]*)?"
    r"(?:\((?:0|[1-9]\d{0,2}-\d{1,4}|0?\d{2,4}(?:-\d{1,2})?)\)" + _SEP + r")?"
    r"\d(?:" + _SEP + r"\d){4,}" + _PHONE_END, re.I,
)
_SHORT_PHONE_LABEL_RE = re.compile(
    r"(?:\b(?:tel(?:efon\w*)?(?:(?:\.[ \t]*|[ \t]+)(?:kom[oó]rk\w*|lokaln\w*|stacjonarn\w*))?|fax|faks(?:em)?|phone|"
    r"toll[ -]free|numer[ \t]+bezpłatny)|[☎☏📞📱])[.: \t/]{0,8}\Z|"
    r"\binfolini[^\W\d_]*[^\d\n]{0,80}:[ \t]*\Z|" + _GLUED_PHONE_LABEL + r"[ \t]*\Z", re.I)
_PHONE_LABEL_RE = re.compile(
    r"\b(?:tel(?:efon[^\W\d_]*)?|fax|faks(?:em)?|kom(?:[oó]rk[^\W\d_]*)?|gsm|"
    r"infolini[^\W\d_]*|(?:za)?dzwo[ńn][^\W\d_]*|Blikiem|"
    r"lini[ęaąie][ \t]+wsparcia)(?![^\W\d_])|"
    r"\bkontakt(?=tel[.:])|[☎☏📞📱]", re.I,
)
_PHONE_BREAK_RE = re.compile(
    r"\n|\.[ \t]+(?=\d)|[ \t]+(?=\(?(?:[01]?\d|2[0-3])[.:][0-5]\d"
    r"|\d{1,2}[.)][ \t]+\d{1,2}[./]\d{1,2}[./](?:19|20)\d{2})")
_SECTION_MARKER_RE = re.compile(r"\d{1,2}[.)](?!\d)")
_PHONE_EXTENSION_RE =re.compile(r"(?:(?:[ \t]+,?[ \t]*|,[ \t]*)wew(?:n(?:ętrzny)?)?\.?[ \t]*\d{1,5}|[ \t]+do[ \t]+\d{1,3}(?=[ \t]*(?:[.;,](?!\d)|\r?\n|\Z)))(?!\w|[ \t]*\d)", re.I)
_SERVICE_PHONE_RE = re.compile(r"(?<![\w+])[1-9]\d{2}(?![\w\d]|[ \t./()-]*\d)")
_SERVICE_PHONE_LABEL_RE = re.compile(
    r"(?:\b(?:tel(?:efon\w*)?|phone)[.: \t]{0,8}(?:alarmow\w*[.: \t]{0,8})?|"
    r"\b(?:numer[ \t]+)?bezpłatny[.: \t]{0,8}|"
    r"\b(?:bezpłatny[ \t]+)?numer[ \t]+alarmowy[.: \t]{0,8}|"
    r"(?:^|\n)[ \t]*(?:Straż pożarna|Policja|Pogotowie|Ratunek|Lekarz pogotowia ratunkowego)"
    r"(?:[ \t]+\([^()\n]{1,60}\))?[ \t]*:[ \t]*)"
    r"(?:[1-9]\d{2}[ \t]*(?:[,;]|i|lub|oraz)[ \t]*)*\Z", re.I)
_OTHER_NUMBER_LABEL_RE = re.compile(r"\b(?:NIP|REGON|PESEL|KRS|ISBN|kod)\b[^\d\n]{0,20}\Z", re.I)
_PHONE_LIST_HEADING_RE = re.compile(
    r"[ \t]*(?:numery[ \t]+kontaktowe|telefony(?:[ \t]+(?:kontaktowe|zaufania))?|fax|faks|zapisz[ \t]+się|"
    r"(?:[^.!?\n]{0,40}\?[ \t]*)?(?:za)?dzwoń!?)"
    r"(?:[ \t]*\([^()\n]{1,80}\))?[ \t]*:?[ \t]*", re.I)
_CONTACT_INTRO_RE = re.compile(
    r"\b(?:kontakt\b|zapisy\b|zapisz[ \t]+się\b|rejestracja\b)"
    r"(?:nr\.|[^\n.!?;]){0,75}\Z", re.I)
_PHONE_HEADING_RE = re.compile(
    r"(?<!\[)\b(?:tel(?:efon[^\W\d_]*)?\b|kontakt\b|zapisy\b|zapisz[ \t]+się\b|"
    r"(?:za)?dzwo[ńn][^\W\d_]*\b|(?:pod|na)[ \t]+numer(?:em)?)"
    r"(?:tj\.|nr\.|[^\d\n.!?;\[\]]){0,75}:?[ \t]*(?:\n[ \t]*){1,4}\Z", re.I)
_CONTACT_EXCLUDE_RE = re.compile(r"\b(?:spraw\w*|kod\w*|kwot\w*|statystyk\w*|taryf\w*)\b", re.I)
_PHONE_LINE_RE = re.compile(r"(?<!\w)\d{2,3}(?:[ \t-]\d{2,4}){2,3}(?!\w)")
_PHONE_TITLE_RE = re.compile(r"\b(?:film\w*|serial\w*|tytuł\w*|zatytułowan\w*)\b", re.I)
_STAFF_ROLE_RE = re.compile(
    r"\b(?:inspektor|referent|koordynator|psycholog|księgowość|księgowy|księgowa|"
    r"pracownicy[ \t]+socjalni|łowczy|podłowczy)\b", re.I)


def _phone_list_context(text: str, start: int) -> bool:
    # ponytail: 2,000-character lookback and 400-character entry suffix; longer
    # lists need repeated headings or a section parser. Never cross prose.
    end = text.find('\n', start, start + 401)
    if end < 0 and len(text) > start + 400:
        return False
    lines = text[max(0, start - 2000):end if end >= 0 else len(text)].splitlines()
    if start > 2000:
        lines = lines[1:]  # A truncated line cannot establish a heading.
    for i, line in enumerate(reversed(lines)):
        if _PHONE_LIST_HEADING_RE.fullmatch(line):
            return i > 0
        if not line.strip():
            continue
        phone = _PHONE_RE.search(line)
        if not phone or not _phone_ok(phone[0], short=True) or _is_amount(line, *phone.span()):
            return False
        if i == 0 and phone.start() != start - (text.rfind('\n', 0, start) + 1):
            return False  # Numbers in an entry's description are not list items.
        before, after = line[:phone.start()].strip(), line[phone.end():].strip()
        if _OTHER_NUMBER_LABEL_RE.search(before):
            return False
        # Either "City: number" or "number – helpline description". Bare
        # numeric rows are ambiguous even below an earlier contact heading.
        named = re.fullmatch(r"[^\W\d_][^\d\n:;]{0,79}[:–—-]", before)
        described = before in ('', '•', '-', '*') and re.match(r"[–—-][ \t]+[^\W\d_]", after)
        dialling = re.fullmatch(r"[-•*]?[ \t]*(?:dzwoniąc[ \t]+)?z[ \t]+[^\W\d_][^\d\n:;.!?]{0,100}", before, re.I)
        if not (((named or dialling) and after in ('', ',', ';', '.')) or described):
            return False
    return False


def _phone_context(text: str, start: int, raw: str) -> bool:
    # A previous redaction is a boundary, not a fresh "Telefon" cue. Include
    # one marker's extra width so the 75-character window cannot bisect it.
    before = re.split(r"\n[ \t]*\n|\[Telefon\]|\[PII\]",
                      text[max(0, start - 75 - len(PHONE_TAG)):start])[-1][-75:]
    if _OTHER_NUMBER_LABEL_RE.search(before):
        return False
    intro = _CONTACT_INTRO_RE.search(before)
    if intro and not _CONTACT_EXCLUDE_RE.search(intro[0]):
        return True
    # A contact email must not erase an explicit contact label for the phone
    # following it. Other redactions still form boundaries.
    if re.search(r'\bkontakt[ \t]*:[ \t]*\[PII\][ \t,;]*\Z',
                 text[max(0, start-75):start], re.I):
        return True
    lead = re.split(r'\[Telefon\]|\[PII\]', text[max(0, start-240):start])[-1]
    # "Pod numerem" also introduces contract/case references. Require a
    # communication cue when the number has no explicit telephone label.
    number_contact = bool((_PHONE_LABEL_RE.search(lead) or re.search(
        r'\b(?:informacj[^\W\d_]*|zgłosz[^\W\d_]*|zapis[^\W\d_]*|SMS(?:-a)?|połączeni[^\W\d_]*)\b', lead, re.I))
        and not _CONTACT_EXCLUDE_RE.search(lead)
        and not re.search(r'\b(?:umow|faktur|dokument)[^\W\d_]*\b', lead, re.I))
    heading = _PHONE_HEADING_RE.search(text[max(0, start-160):start])
    if (heading and not _CONTACT_EXCLUDE_RE.search(heading[0])
            and (not re.match(r'(?:pod|na)\b', heading[0], re.I)
                 or _PHONE_LABEL_RE.search(heading[0]) or number_contact)):
        return True
    if (number_contact and re.search(r'\b(?:pod[ \t]+numerem|na[ \t]+numer)[ \t]*:?[ \t]*\Z', before, re.I)):
        return True
    if re.search(r'\bтелефону[ \t]*:[ \t]*\Z', before, re.I):
        return True
    # An international phone may precede a labelled fax in the same contact list.
    # Keep this cue adjacent: arbitrary later contact prose is not evidence.
    after = text[start + len(raw):start + len(raw) + 40]
    if re.match(r'[ \t]*[–—-][ \t]+telefon\b', after, re.I):
        return True
    if (re.match(r"\(?(?:\+|00)", raw)
            and re.match(r"[ \t]*[,;][ \t]*(?:fax|faks(?:em)?)[.: \t]*(?=[+(0-9])", after, re.I)):
        return True
    # A foreign-looking +number can also be an increment in a financial table.
    return bool(re.match(r"\(?(?:\+|00)[ \t]*48|\+\d{1,3}[ \t]+\(0\)", raw)
                or _PHONE_LABEL_RE.search(before))


def _directory_context(text: str, start: int, end: int) -> bool:
    """Recognize phones in bounded staff and named-address contact records."""
    raw = text[start:end]
    foreign = raw.startswith('+') and _phone_ok(raw)
    if not foreign and (not _pl_national_ok(_digits(raw)) or not re.search(r'[ \t-]', raw)):
        return False
    # ponytail: local 1,600-character directory evidence; longer isolated rows
    # need preserved upstream table structure, not an unbounded document cue.
    left, right = max(0, start-800), min(len(text), end+800)
    nearby = text[left:right]
    line_start, line_end = text.rfind('\n', 0, start)+1, text.find('\n', end)
    line_end = len(text) if line_end < 0 else line_end
    before, after = text[line_start:start], text[end:line_end]
    if _CONTACT_EXCLUDE_RE.search(before) or _OTHER_NUMBER_LABEL_RE.search(before):
        return False
    if foreign:
        # A comma-delimited name/address entry needs a labelled phone nearby;
        # a +number alone could be an increment in a financial table.
        address = re.search(r'(?:^|,)[ \t]*[^\W\d_][^,\n]{1,120},'
                            r'[^\n]{0,160}\d[^\n]{0,80},[ \t]*\Z', before)
        return bool(address and any(
            _SHORT_PHONE_LABEL_RE.search(nearby[max(0,m.start()-80):m.start()])
            for m in _PHONE_RE.finditer(nearby) if m[0].startswith('+') and _phone_ok(m[0])))
    standalone = not before.strip() and not after.strip()
    lead = text[max(0, start-400):start]
    if (standalone and re.search(r'\b(?:infolini[^\W\d_]*|helpline)\b', lead, re.I)
            and not _CONTACT_EXCLUDE_RE.search(lead)):
        return True
    if (standalone and re.search(r'\bnr\.?[ \t]+telefonu\b', nearby, re.I)
            and re.search(r'\b(?:pok[oó]j|pokoju)\b', nearby, re.I)
            and re.search(r'\bnazwisko\b', nearby, re.I)):
        return True
    # Email/name/role records and phone-led service directories repeat formatted
    # values. A lone staff title beside a number is insufficient evidence.
    phones = [m for m in _PHONE_LINE_RE.finditer(nearby)
              if _pl_national_ok(_digits(m[0])) and not _is_amount(nearby, *m.span())]
    roles = list(_STAFF_ROLE_RE.finditer(nearby))
    if len(phones) < 2 or len(roles) < 2:
        return False
    return bool((re.search(r'\[PII\]', before[-100:] + after[:100]) and
                 re.search(r'[^\W\d_]', before[-100:]))
                or (not before.strip() and re.match(r'[ \t]+[^\W\d_]', after))
                or _STAFF_ROLE_RE.search(before[-100:]))


# Amounts can look like phones or checksum-valid national IDs. Keep the
# currency adjacent to the number, allowing a decimal, multiplier and line wrap.
_MONEY_GAP = r"[ \t]*(?:\n[ \t]*)?"
_CURRENCY = r"(?:PLN|z[łl](?:ot(?:ych|ego|emu|ymi|ym|y|e))?|EUR(?:O)?|USD|GBP|CHF)\b|[%€$£]"
_MONEY_AFTER_RE = re.compile(
    r"(?:[,.][0-9]+)?" + _MONEY_GAP
    + r"(?:(?:tys\.?|mln|mld|bln)\b\.?" + _MONEY_GAP + r")?"
    + r"(?:" + _CURRENCY + r")", re.I,
)
_MONEY_BEFORE_RE = re.compile(r"(?:\b(?:PLN|EUR|USD|GBP|CHF)|[€$£])" + _MONEY_GAP + r"$", re.I)


def _is_amount(text: str, start: int, end: int) -> bool:
    return bool(_MONEY_AFTER_RE.match(text, end) or _MONEY_BEFORE_RE.search(text[max(0, start - 32):start]))


def _digits(s: str) -> str:
    return re.sub(r"\D", "", s)


def _pesel_ok(d: str) -> bool:
    if len(d) != 11 or not d.isdigit():
        return False
    weights = (1, 3, 7, 9, 1, 3, 7, 9, 1, 3)
    check = sum(w * int(x) for w, x in zip(weights, d[:-1]))
    if str((10 - check % 10) % 10) != d[-1]:
        return False
    yy, mm, dd = int(d[0:2]), int(d[2:4]), int(d[4:6])
    century = {0: 1900, 1: 2000, 2: 2100, 3: 2200, 4: 1800}.get(mm // 20)
    if century is None:
        return False
    try:
        dt.date(century + yy, mm % 20, dd)
    except ValueError:
        return False
    return True


def _nip_ok(d: str) -> bool:
    if len(d) != 10 or not d.isdigit():
        return False
    weights = (6, 5, 7, 2, 3, 4, 5, 6, 7)
    rem = sum(w * int(x) for w, x in zip(weights, d[:-1])) % 11
    return rem != 10 and rem == int(d[-1])


def _regon_ok(d: str) -> bool:
    if not d.isdigit() or len(d) not in (9, 14):
        return False
    weights = ((8, 9, 2, 3, 4, 5, 6, 7) if len(d) == 9
               else (2, 4, 8, 5, 0, 9, 7, 3, 6, 1, 2, 4, 8))
    rem = sum(w * int(x) for w, x in zip(weights, d[:-1])) % 11
    if rem == 10:
        rem = 0
    return rem == int(d[-1])


def _land_register_ok(court: str, number: str, check: str) -> bool:
    code = court + number.zfill(8)
    if not all(c in _LAND_REGISTER_VALUES for c in code):
        return False
    weights = (1, 3, 7) * 4
    return sum(w * _LAND_REGISTER_VALUES[c] for w, c in zip(weights, code)) % 10 == int(check)


def _iban_ok(raw: str) -> bool:
    # Do not assemble an alphanumeric account from neighbouring prose words.
    # Compact and ordinary four-character printed groups remain supported.
    groups = re.split(r'[ \t-]+', raw)
    if len(groups) > 1 and any(re.search(r'[A-Za-z]{5}', group) for group in groups):
        return False
    compact = re.sub(r"[\s-]+", "", raw).upper()
    if compact.isdigit() and len(compact) == 26:
        compact = "PL" + compact
    if not re.fullmatch(r"[A-Z]{2}[0-9]{2}[A-Z0-9]+", compact):
        return False
    if len(compact) != _IBAN_LENGTHS.get(compact[:2]) or not 2 <= int(compact[2:4]) <= 98:
        return False
    if compact.startswith("PL") and not compact[2:].isdigit():
        return False
    rearranged = compact[4:] + compact[:4]
    nums = "".join(str(ord(c) - 55) if c.isalpha() else c for c in rearranged)
    return int(nums) % 97 == 1


def _tag(start, end, tag):
    return tag


def _replace_epuap(text: str, *, mask=_tag) -> tuple[str, int]:
    last_end, count = None, 0

    def replace(m):
        nonlocal last_end, count
        labelled = _EPUAP_LABEL_RE.search(text, max(0, m.start()-64), m.start())
        continued = last_end is not None and re.fullmatch(r'[ \t]*[;,][ \t]*|[ \t]+(?:lub|albo|i|oraz)[ \t]+', text[last_end:m.start()], re.I)
        if not labelled and not continued:
            return m[0]
        last_end = m.end()
        count += 1
        return mask(m.start(), m.end(), PII_TAG)

    out = _EPUAP_RE.sub(replace, text)
    return out, count


def _replace_documents(text: str, *, mask=_tag) -> tuple[str, int]:
    n = 0

    def replace(m):
        nonlocal n
        number = re.sub(r"[ \t-]", "", m["number"]).upper()
        passport = m["label"].lower().startswith("paszport")
        if passport:
            short_diplomatic = ('dyplomatyczn' in m['label'].lower()
                                and re.search(r'\b(?:nr|numer)\b', m['label'], re.I)
                                and re.fullmatch(r'\d{4,5}', number))
            if not short_diplomatic and not re.fullmatch(r"[A-Z]{2}[0-9OIL]{7}|[A-Z][0-9]{6,9}|[0-9]{6,12}", number):
                return m[0]
        elif not re.fullmatch(r"[A-Z]{3}[0-9OIL]{6}", number):
            return m[0]
        n += 1
        return m["label"] + mask(m.start('number'), m.end('number'), PII_TAG)

    # Passport detection is label/format based; no MRZ check digit is present here.
    return _DOCUMENT_RE.sub(replace, text), n


def _replace_land_registers(text: str, *, mask=_tag) -> tuple[str, int]:
    n = 0

    def replace(m):
        nonlocal n
        # A label covers shortened, spaced or mistyped forms, as for labelled
        # PESEL/NIP. Unlabelled numbers need the full form and a valid check digit.
        labelled = _LAND_REGISTER_LABEL_RE.search(text, max(0, m.start()-64), m.start())
        full = re.fullmatch(r"[A-Z]{2}\d[A-Z]/\d{8}/\d", m[0])
        if not labelled and not (full and _land_register_ok(m['court'], m['number'], m['check'])):
            return m[0]
        n += 1
        return mask(m.start(), m.end(), PII_TAG)

    return _LAND_REGISTER_RE.sub(replace, text), n


def _pl_national_ok(d: str) -> bool:
    return len(d) == 9 and d.isdigit() and d[:2] in _PL_PREFIX


def _phone_ok(raw: str, short: bool = False) -> bool:
    s = raw.strip()
    if _VANITY_PHONE_RE.fullmatch(s):
        return short
    if s.startswith('/') or s.endswith('/'):
        return False
    s = re.sub(r'^\(\+', '+', s)
    s = re.sub(r'^\(00\)[ \t]*', '00', s)
    if re.fullmatch(r"\d{4}[-./]\d{2}[-./]\d{2}", s):
        return False
    if re.fullmatch(r'\(?(?:[01]?\d|2[0-3])[.:][0-5]\d[ \t]*[–—-][ \t]*(?:[01]?\d|2[0-3])[.:][0-5]\d\)?', s):
        return False
    if re.fullmatch(r"\d{2}-\d{3}", s):  # postal code
        return False
    # Ministry / court file numbers: BPRM.4820.2.3.2020, LUB-OMK.601.1.2024.3.
    # They never start with "+"; dotted international phones do: +218.21.555.0123.
    if not s.startswith('+') and (
            s.count(".") >= 3 or (s.count(".") >= 1 and re.search(r"(?<!\d)20\d{2}(?!\d)", s))):
        return False
    if re.search(r"(?<!\d)\d{1,2}[-./]\d{1,2}[-./](?:19|20)\d{2}(?!\d)", s):
        return False
    d = _digits(raw)
    if short and re.fullmatch(r'[2-9]\d{2}-[2-9]\d{2}-\d{4}', s):
        return True
    if short and s.startswith('(0)'):
        return 7 <= len(d) <= 12
    # ponytail: label + shape + length, not a country numbering-plan validator.
    if short and re.match(r'^\(0?\d{2,4}(?:-\d{1,2})?\)', s):
        return 8 <= len(d) <= 15
    # EUR-Lex puts country and area code in parentheses: "(32-2) 299 11 11" is +32 2 299 11 11.
    s = re.sub(r'^\(([1-9]\d{0,2})-(\d{1,4})\)', r'+\1 \2', s)
    if s.startswith(('+', '00')):
        international = _digits(s.replace('(0)', ''))
        if s.startswith('00'):
            international = international[2:]
        if not international.startswith('48'):
            # ponytail: explicit prefix + length, not a global numbering-plan
            # validator. Add country metadata if review finds false positives.
            return (7 if '(0)' in s else 8) <= len(international) <= 15 and international[0] != '0'
    if d.startswith("00"):
        d = d[2:]
    if d.startswith("48") and len(d) >= 11:
        rest = d[2:]
        if rest.startswith("0"):
            rest = rest[1:]
        return _pl_national_ok(rest)
    if d.startswith("0") and len(d) == 11 and d[:2] in {"01", "02", "07"}:
        return True
    if d.startswith("0") and len(d) >= 10:
        return _pl_national_ok(d.lstrip("0")) or (short and len(d) <= 12)
    return (_pl_national_ok(d)
            or (short and 5 <= len(d) <= 8 and d[0] != '0')
            or (short and 9 <= len(d) <= 12 and bool(re.fullmatch(r'\d{2,4}(?:[ \t-]\d{2,4}){2,3}', s)))
            or (short and 9 <= len(d) <= 12 and bool(re.fullmatch(r'\d{2,4}\.\d{6,8}', s)))
            or (short and 9 <= len(d) <= 12 and d.startswith('0')))


def _trim_glued_email_prefix(text: str, start: int, end: int) -> int:
    """Drop a title-case letter prefix when the remainder is already a complete email.

    Prefix-only: do not touch dotted locals or suffix glue. Leftmost 5–11 letters.
    Remainder local must be lowercase letters, length at least 5.
    """
    frag = text[start:end]
    at = frag.find('@')
    if at < 0:
        return start
    loc = frag[:at]
    if '.' in loc or '-' in loc or sum(c.isupper() for c in loc) != 1:
        return start
    for i in range(5, min(12, at)):
        prefix, rest = frag[:i], frag[i:]
        if not prefix.isalpha() or not (prefix[0].isupper() and prefix[1:].islower()):
            continue
        r_at = rest.find('@')
        rloc = rest[:r_at] if r_at >= 0 else ''
        if r_at < 5 or not rloc.isalpha() or not rloc.islower():
            continue
        if _EMAIL_RE.fullmatch(rest):
            return start + i
    return start


def _glued_email_trim_starts(text: str) -> dict[int, int]:
    """Map match starts to trimmed starts when a passage has two or more glues.

    A lone title-case local is a real mailbox (Lostandfound@…). Two or more
    CELEX-shaped prefix+mailbox tokens in one passage are column glue.
    """
    starts = {}
    for pattern in (_EMAIL_WRAP_RE, _EMAIL_RE):
        for m in pattern.finditer(text):
            new = _trim_glued_email_prefix(text, m.start(), m.end())
            if new != m.start():
                starts.setdefault(m.start(), new)
    return starts if len(starts) >= 2 else {}


def _replace_checked(text: str, pattern: re.Pattern, tag: str, ok,
                     label: re.Pattern | None = None, *, mask=_tag) -> tuple[str, int]:
    n = 0
    trims = _glued_email_trim_starts(text) if pattern in (_EMAIL_RE, _EMAIL_WRAP_RE) else {}

    def _sub(m):
        nonlocal n
        if label is not None and not label.search(text, max(0, m.start() - 64), m.start()):
            return m.group(0)
        if pattern is _PESEL_RE and re.search(r'[/?&][^\s]*\Z', text[max(0, m.start()-500):m.start()]):
            return m.group(0)  # Bare URL path/query digits are not personal identifiers.
        if not ok(m.group(0)) or (pattern not in (_EMAIL_RE, _EMAIL_WRAP_RE, _EMAIL_FRAGMENT_RE)
                                 and _is_amount(text, m.start(), m.end())):
            return m.group(0)
        start = trims.get(m.start(), m.start())
        n += 1
        return text[m.start():start] + mask(start, m.end(), tag)

    return pattern.sub(_sub, text), n


def _replace_phones(text: str, *, mask=_tag) -> tuple[str, int]:
    n = 0
    out = []
    pos = 0
    last_phone_end = None
    last_phone_complete = False
    while True:
        m = _PHONE_RE.search(text, pos)
        if not m:
            out.append(text[pos:])
            break
        raw = m[0]
        end = m.end()
        replacement = raw
        line_end = text.find('\n', m.start(), end)
        if line_end >= 0 and re.fullmatch(r'\d{1,3}', text[m.start():line_end]):
            next_end = text.find('\n', line_end+1, end)
            if _directory_context(text, line_end+1, next_end if next_end >= 0 else end):
                # A flattened table's room cell is not a phone country prefix.
                out.append(text[pos:line_end+1])
                pos = line_end+1
                continue
        immediate_phone_label = bool(_SHORT_PHONE_LABEL_RE.search(text[max(0, m.start()-100):m.start()]))
        short = immediate_phone_label
        short = short or (re.fullmatch(r'116[ \t]?\d{3}', raw)
                          and _phone_context(text, m.start(), raw))
        short = short or _phone_list_context(text, m.start())
        novel_country_area = (bool(re.match(r'^\([1-9]\d{0,2}-\d{1,4}\)', raw))
                              and not re.match(r'^\(0?\d{2,4}(?:-\d{1,2})?\)', raw))
        # Broadened country-area forms require an immediate strong label and
        # reject numeric continuations, slash lists, and nearby title prose.
        novel_country_area_ok = not novel_country_area or (
            immediate_phone_label
            and not re.search(r'\r?\n[ \t]*\d|/', raw)
            and not re.match(r'[ \t]*(?:\r?\n[ \t]*\d|/)', text[end:])
            and not _PHONE_TITLE_RE.search(text[max(0, m.start()-100):m.start()])
        )
        # A bare five-digit continuation followed by a place/name can be a
        # postal address. It needs its own phone label to override that ambiguity.
        postal = re.fullmatch(r'\d{5}', raw) and re.match(r'[ \t]+[^\W\d_]', text[end:])
        # So is a 5-6 digit number after a complete phone and a bare line break: a table
        # cell, "Fax (32-2) 287 25 24\n28 648,00". After a comma a list goes on,
        # "95-1-511098,\n514262"; after a fragment the next line is its wrapped rest,
        # "tel. +48 609 \n953 709", or the next local number, "(30) 25 10 22 33 25,\n22 33 28".
        cell = (last_phone_complete and 5 <= len(_digits(raw)) <= 6
                and re.fullmatch(r'[ \t]*\n[ \t\n]*', text[last_phone_end:m.start()]))
        if (last_phone_end is not None and not postal and not cell
                and re.fullmatch(r"[ \t\n,;]{1,12}|[ \t]+(?:lub|albo)[ \t]+", text[last_phone_end:m.start()], re.I)):
            short = True
        if (last_phone_end is not None
                and re.fullmatch(r'(?:[ \t]+(?:w sprawie\b|\((?:w godzinach|dostępny)\b)'
                                 r'|[ \t]*[–—-][ \t]+)[^\d\n.!?]{1,200}\n[ \t\n]{0,8}',
                                 text[last_phone_end:m.start()], re.I)
                and _pl_national_ok(_digits(raw))):
            short = True
        if (novel_country_area_ok and (short or _phone_context(text, m.start(), raw)
                or _directory_context(text, m.start(), line_end if line_end >= 0 else end))):
            if (_is_amount(text, m.start(), end)
                    and re.fullmatch(r'\d{1,3}(?:\.[ \t]*\d{3})+', raw)):
                out.extend((text[pos:m.start()], raw))
                pos = end
                continue
            # Stop before a new line or opening hours, but only after a complete
            # phone. Look past the greedy match's last digit to recognize HH:MM.
            for gap in _PHONE_BREAK_RE.finditer(text, m.start(), end + 6):
                if gap.start() >= end:
                    break
                # Foreign lengths vary: a valid-looking prefix may still be an
                # incomplete wrapped number. Only a section marker ends it.
                if (raw.startswith(('+', '00')) and not re.match(r'(?:\+|00)[ \t]*48', raw)
                        and gap[0] == '\n'
                        and not _SECTION_MARKER_RE.match(text, gap.end())):
                    continue
                prefix = text[m.start():gap.start()].rstrip(". \t")
                # A short number also ends at a section marker: "albo 1234567\n2. Szkoła".
                section = gap[0] == '\n' and bool(_SECTION_MARKER_RE.match(text, gap.end()))
                if _phone_ok(prefix, short and section) and not _is_amount(text, m.start(), gap.start()):
                    end = m.start() + len(prefix)
                    raw = prefix
                    replacement = raw
                    break
            if _is_amount(text, m.start(), end):
                out.extend((text[pos:m.start()], raw))
                pos = end
                continue
            # A suffix range repeats the final extension digits, not a second
            # complete phone. Only accept it after a valid full base number.
            extension = re.fullmatch(r"(.+?)[/-](\d{1,3})", raw)
            parenthesized_extension = re.fullmatch(r'(.+?)[ \t]+\(\d{1,5}', raw)
            if not (text[end:end+1] == ')' and parenthesized_extension
                    and _phone_ok(parenthesized_extension[1])):
                parenthesized_extension = None
            pair = re.fullmatch(r"(.+?)[ \t]+[–—-][ \t]+(.+)", raw)
            shared = re.match(r"\(\d{2,4}\)[ \t]*", pair[1]) if pair else None
            full_range = (pair and _phone_ok(pair[1], short)
                          and _phone_ok((shared[0] if shared else '') + pair[2], short))
            if (_phone_ok(raw, short) or (extension and _phone_ok(extension[1]))
                    or parenthesized_extension or full_range):
                if raw.count('(') > raw.count(')') and text[end:end+1] == ')':
                    end += 1
                suffix = _PHONE_EXTENSION_RE.match(text, end)
                if suffix and not _is_amount(text, suffix.start(), suffix.end()):
                    end = suffix.end()
                start = m.start()
                # Parentheses enclosing prose are punctuation, not phone syntax.
                if raw.startswith('(+') and raw.count('(') > raw.count(')') and text[end-1:end] != ')':
                    start += 1
                replacement = text[m.start():start] + mask(start, end, PHONE_TAG)
                n += 1
            # ponytail: split only pairs (<=22 digits); longer lists need label
            # context to distinguish them from accounts. Never redact ID tails.
            elif 18 <= len(_digits(raw)) <= 22 and "." not in raw:
                # A slash separates alternatives, the second without the shared
                # prefix: "(31-30) 274 44 13/274 44 01". Try slashes first.
                gaps = sorted(re.finditer(r"[ \t]*/[ \t]*|[ \t\n]+", raw), key=lambda g: '/' not in g[0])
                for gap in gaps:
                    if _phone_ok(raw[:gap.start()]) and _phone_ok(raw[gap.end():], '/' in gap[0]):
                        replacement = (mask(m.start(), m.start()+gap.start(), PHONE_TAG)
                                       + gap[0] + mask(m.start()+gap.end(), end, PHONE_TAG))
                        n += 2
                        break
        out.extend((text[pos:m.start()], replacement))
        if replacement != raw:
            last_phone_end = end
            last_phone_complete = _phone_ok(raw)
        pos = end
    return "".join(out), n


def _replace_extensions(text: str, *, mask=_tag) -> tuple[str, int]:
    count = 0

    def replace(m):
        nonlocal count
        before = text[max(0, m.start()-500):m.start()]
        # Only a telephone followed by extension/name entries establishes scope.
        heading = re.search(r'\btel(?:efon)?[.: \t]*\[Telefon\]'
                            r'(?P<entries>(?:\s*wewn?\.?[ \t]+\d{1,5}[ \t]+[^\d\n]+)*\s*)\Z', before, re.I)
        if not heading or _is_amount(text, m.start('number'), m.end('number')):
            return m[0]
        count += 1
        return m['label'] + mask(m.start('number'), m.end('number'), PHONE_TAG)

    output = re.sub(r'(?P<label>(?:^|\n)[ \t]*wewn?\.?[ \t]+)(?P<number>\d{1,5})(?!\w)',
                    replace, text, flags=re.I)
    return output, count


def scrub_pii(text: str, *, spans: list | None = None) -> tuple[str, dict[str, int]]:
    """Return (scrubbed_text, counts); optionally append exact original spans.

    Offset tracking is opt-in; corpus ingestion keeps its existing return type
    and does not allocate a character map. Unicode offsets count code points.
    """
    counts = {k: 0 for k in COUNTS}
    if not text:
        return text, counts
    original = text
    text = text.translate(_SPACES)  # One character per character: offsets survive.
    offsets = list(range(len(text))) if spans is not None else None
    edits = []

    def mask(start, end, tag):
        if spans is not None:
            a, b = offsets[start], offsets[end-1] + 1
            spans.append(dict(start=a, end=b, text=original[a:b],
                              label='phone' if tag == PHONE_TAG else 'pii'))
            edits.append((start, end, tag))
        return tag

    def apply(replacer, *args):
        nonlocal text
        text, result = replacer(text, *args, mask=mask)
        # Each replacement pass uses one coordinate system. Apply its edits
        # backwards only after all regex callbacks have finished.
        for start, end, tag in reversed(edits):
            offsets[start:end] = [offsets[start]] * len(tag)
        edits.clear()
        return result

    # Longest / most specific first so a 26-digit account is not sliced
    # into REGON / PESEL / NIP / phone. Checksums live in the replace callback.
    def pins(value, *, mask):
        def replace(m):
            if _is_amount(value, m.start('number'), m.end('number')):
                return m[0]
            counts['pin'] += 1
            return m['label'] + mask(m.start('number'), m.end('number'), PII_TAG)
        return _PIN_RE.sub(replace, value), None
    apply(pins)
    foreign = apply(_replace_checked, _IBAN_RE, PII_TAG, _iban_ok)
    bare = apply(_replace_checked, _ACCOUNT_RE, PII_TAG, _iban_ok)
    counts["account"] = foreign + bare
    counts['electronic_address'] = apply(_replace_checked, _EDELIVERY_RE, PII_TAG, lambda _: True)
    counts['electronic_address'] += apply(_replace_epuap)
    counts['krs'] = apply(_replace_checked, _KRS_RE, PII_TAG, lambda _: True, _KRS_LABEL_RE)
    counts["document"] = apply(_replace_documents)
    counts["land_register"] = apply(_replace_land_registers)
    def labelled_ids(value, *, mask):
        def replace(m):
            if len(_digits(m['number'])) < 7 or _is_amount(value, m.start('number'), m.end()):
                return m[0]
            counts[m['kind'].lower()] += 1
            return m['label'] + mask(m.start('number'), m.end('number'), PII_TAG)
        return _LABELLED_ID_RE.sub(replace, value), None
    apply(labelled_ids)
    n14 = apply(_replace_checked, _REGON14_RE, PII_TAG, _regon_ok, _REGON_LABEL_RE)
    n9 = apply(_replace_checked, _REGON9_RE, PII_TAG, _regon_ok, _REGON_LABEL_RE)
    counts["regon"] += n14 + n9
    n_dash = apply(_replace_checked, _NIP_DASH_RE, PII_TAG, lambda s: _nip_ok(_digits(s)), _NIP_LABEL_RE)
    n_plain = apply(_replace_checked, _NIP_RE, PII_TAG, lambda s: _nip_ok(_digits(s)), _NIP_LABEL_RE)
    counts["nip"] += n_dash + n_plain
    # The domain break is the last newline; a hyphenated local part may add an earlier one.
    counts["email"] = apply(_replace_checked, _EMAIL_WRAP_RE, PII_TAG,
        lambda s: not _EMAIL_RE.fullmatch(s[:s.rindex('\n')].rstrip('. \t\r')))
    counts["email"] += apply(_replace_checked, _EMAIL_RE, PII_TAG, lambda _: True)
    counts["email"] += apply(_replace_checked, _EMAIL_FRAGMENT_RE, PII_TAG,
                             lambda _: True, _EMAIL_LABEL_RE)
    counts["phone"] = apply(_replace_phones)
    counts["phone"] += apply(_replace_checked, _SERVICE_PHONE_RE, PHONE_TAG,
                             lambda _: True, _SERVICE_PHONE_LABEL_RE)
    counts['phone'] += apply(_replace_extensions)
    # Contact context takes precedence over coincidental PESEL checksums in
    # foreign phone numbers; explicitly labelled identifiers were handled first.
    bare_pesel = apply(_replace_checked, _PESEL_RE, PII_TAG, _pesel_ok)
    counts['pesel'] += bare_pesel
    return text, counts