Upload aztext/tokenize.py with huggingface_hub

aztext/tokenize.py

@@ -0,0 +1,49 @@
+"""Simple, robust word and sentence tokenizers for Azerbaijani.
+
+* :func:`word_tokenize` returns word tokens made of letters (ASCII +
+  Azerbaijani + combining marks) and digits, allowing internal apostrophes and
+  hyphens (e.g. ``Qur'an``, ``elmi-tədqiqat``).  Punctuation is dropped.
+* :func:`sent_tokenize` splits after sentence terminators ``. ! ? …`` while
+  keeping the terminator attached to its sentence.
+
+Both use an explicit Azerbaijani letter class rather than ``\\w`` so behaviour
+does not depend on the platform's Unicode word rules.
+"""
+
+import re
+
+from .alphabet import AZ_LETTERS
+
+# Letter class: ASCII letters + Azerbaijani letters + combining marks.
+_LETTER = "A-Za-z" + re.escape(AZ_LETTERS) + "̀-ͯ"
+
+# A word: one or more letters/digits, optionally joined by a single internal
+# apostrophe (straight or curly) or hyphen to more letters/digits.
+_WORD_RE = re.compile(
+    r"[" + _LETTER + r"0-9]+(?:['’\-][" + _LETTER + r"0-9]+)*"
+)
+
+# Split on whitespace that follows a sentence terminator.
+_SENT_SPLIT_RE = re.compile(r"(?<=[.!?…])\s+")
+
+
+def word_tokenize(text: str) -> list:
+    """Return the list of word tokens in ``text`` (punctuation removed)."""
+    if not text:
+        return []
+    return _WORD_RE.findall(text)
+
+
+def sent_tokenize(text: str) -> list:
+    """Split ``text`` into sentences.
+
+    Sentences are terminated by ``.``, ``!``, ``?`` or ``…`` (one or more) and
+    the terminator stays with its sentence.  A trailing fragment without a
+    terminator is returned as its own sentence.  Empty pieces are dropped.
+    """
+    if not text:
+        return []
+    stripped = text.strip()
+    if not stripped:
+        return []
+    return [s for s in (p.strip() for p in _SENT_SPLIT_RE.split(stripped)) if s]