Add ud=True flag to decode(): converts native Perseus/AGDT output to Universal Dependencies (HamleDT-style COORD/AuxP/Pnom restructuring + XPOS->UPOS+FEATS mapping)

processing_char_bert_joint.py

@@ -4,7 +4,11 @@ Turns one or more already word-tokenized sentences into the model's input tensor
 four character planes as the base CharBertProcessor, plus a `word_id` array pooling characters
 into words, a `cap` plane -- an actual model input for this fine-tuned checkpoint, unlike the
 base model where capitalization is output-only -- and a `seg_id` plane), and decodes the
-model's raw logits back into UD-style XPOS / lemma / UPOS / dependency-arc predictions.
+model's raw logits back into predictions. The model's NATIVE tag/relation inventory is the
+Perseus/AGDT scheme (9-position morphological tags; Prague-style dependency relations like
+SBJ/OBJ/ATR/COORD/AuxP -- the scheme its OGA training treebank uses), not Universal
+Dependencies. Pass `ud=True` to `decode()` for Universal Dependencies-style output instead
+(UPOS/FEATS/deprel) -- see `to_ud()` below for the conversion and its scope/limitations.
 
 Interface: call the processor with `List[str]` (one already-tokenized sentence, e.g.
 `["ὁ", "ἄνθρωπος", "τρέχει", "."]`) or `List[List[str]]` (a batch of such sentences) --
@@ -57,6 +61,225 @@ def _pack_dia(acc, br, iota, diaer):
     return ((acc * 3 + br) * 2 + iota) * 2 + diaer
 
 
+# ==== Universal Dependencies conversion ========================================================
+#
+# The model's native output is the Perseus/AGDT annotation scheme its OGA training treebank
+# uses: a 9-position morphological tag (pos.person.number.tense.mood.voice.gender.case.degree,
+# one letter per position, "-" for n/a) and Prague-style dependency relations (SBJ, OBJ, ATR,
+# ADV, PNOM, PRED, APOS, COORD, Aux*, ATV/AtvV, ExD, MWE, OCOMP) -- NOT Universal Dependencies.
+#
+# This is a solved, documented conversion problem, not something designed from scratch here:
+# both UD_Ancient_Greek-Perseus and UD_Ancient_Greek-PROIEL are themselves automatic
+# conversions of exactly this annotation style, via Zeman & Ramasamy's HamleDT pipeline
+# (Zeman et al. 2014, "HamleDT: Harmonized Multi-Language Dependency Treebank", LRE) --
+# specifically Treex::Block::HamleDT::{GRC::Harmonize,HarmonizePerseus,Udep} and
+# Treex::Tool::PhraseBuilder::PragueToUD (github.com/ufal/treex), plus the morphology decoder
+# Lingua::Interset::Tagset::GRC::Conll (github.com/dan-zeman/interset). The tables and
+# restructuring rules below port that design to our exact tag/relation inventory.
+#
+# Two relations require real TREE restructuring, not just a label rename, because UD attaches
+# function words/conjuncts differently than Prague style:
+#   - COORD: Prague style makes the coordinating conjunction the head, conjuncts its children.
+#     UD makes the FIRST conjunct the head, the conjunction becomes a `cc` dependent of it, and
+#     other conjuncts become `conj` dependents.
+#   - AuxP: Prague style makes the preposition the head of its complement. UD reverses this:
+#     the nominal complement becomes the head, the preposition becomes a `case` dependent of it.
+#   - Pnom (predicate nominal / copula): Prague style makes the copula verb the head with the
+#     nominal attached as Pnom. UD makes the NOMINAL the head, the copula a `cop` dependent,
+#     and reattaches the subject to the nominal instead of the verb.
+#
+# What relation does the PROMOTED node (first conjunct / nominal complement) get in UD, given
+# it used to be a plain dependent of COORD/AuxP? In full AGDT/HamleDT, the coordinator's or
+# preposition's own outbound relation carries the phrase's true external function via a
+# _CO/_AP label suffix (e.g. "Sb_Co"), which HamleDT strips off and hands to the promoted node.
+# Our 24-label inventory has no such suffix mechanism -- verified directly against real training
+# examples in oga_sota.conllu (e.g. "ἀπὸ" and "ἕως" each attach to their external governor with
+# deprel=AuxP verbatim, and "καὶ" attaches with deprel=COORD verbatim, regardless of whether the
+# phrase functions as a locative adjunct, a coordinated subject, or anything else). So the label
+# itself never carries a recoverable function here, and the promoted node gets UD's generic
+# fallback instead of a falsely precise guess: "obl" for AuxP nominals, "dep" for COORD first
+# conjuncts (see Stage 1/2 below).
+#
+# SCOPE: this ports the core, well-documented HamleDT/UD restructuring rules and a straight
+# per-label relabeling for everything else. It does NOT replicate every Ancient-Greek-specific
+# refinement in the full pipeline (e.g. GRC::Harmonize's regex-based negation-particle
+# splitting, or the is_member/shared-modifier _CO/_AP distinction discussed above) -- treat this
+# as a faithful, reasonably-scoped port, not a byte-exact reproduction of the official
+# UD_Ancient_Greek-Perseus release.
+
+# position 1 (POS) -> UD UPOS. 'c' defaults to CCONJ, refined to SCONJ if its converted deprel
+# is "mark" (i.e. it was AuxC); 'v' stays VERB regardless of mood (participles etc. get
+# VerbForm=Part as a feature, not a UPOS change, matching standard UD_Ancient_Greek practice).
+_POS_TO_UPOS = {
+    "a": "ADJ", "c": "CCONJ", "d": "ADV", "g": "PART", "i": "INTJ", "l": "DET",
+    "m": "NUM", "n": "NOUN", "p": "PRON", "r": "ADP", "u": "PUNCT", "v": "VERB", "x": "X",
+}
+_NUMBER = {"d": "Dual", "p": "Plur", "s": "Sing"}
+_TENSE_ASPECT = {   # (Tense, Aspect-or-None); aorist/perfect/future-perfect analysis per
+    "a": ("Past", "Perf"),   # aorist = perfective past
+    "f": ("Fut", None),
+    "i": ("Past", "Imp"),    # imperfect
+    "l": ("Pqp", None),      # pluperfect
+    "p": ("Pres", None),
+    "r": ("Pres", "Perf"),   # perfect = present relevance of a completed action
+    "t": ("Fut", "Perf"),    # future perfect
+}
+_MOOD_VERBFORM = {   # (Mood-or-None, VerbForm)
+    "i": ("Ind", "Fin"), "s": ("Sub", "Fin"), "o": ("Opt", "Fin"), "m": ("Imp", "Fin"),
+    "n": (None, "Inf"), "p": (None, "Part"), "g": (None, "Gdv"),
+}
+_VOICE = {"a": "Act", "m": "Mid", "p": "Pass", "e": "Mid"}   # e = mediopassive, approximated as Mid
+_GENDER = {"c": "Com", "f": "Fem", "m": "Masc", "n": "Neut"}
+_CASE = {"a": "Acc", "d": "Dat", "g": "Gen", "n": "Nom", "v": "Voc"}
+_DEGREE = {"c": "Cmp", "s": "Sup"}
+
+
+def xpos_to_upos_feats(xpos: str) -> tuple[str, dict]:
+    """9-position Perseus/AGDT tag (e.g. "v3spia---") -> (UD UPOS, UD FEATS dict).
+    "-" in any position means not applicable/unspecified and is simply omitted from FEATS."""
+    pos = xpos[0] if xpos else "x"
+    upos = _POS_TO_UPOS.get(pos, "X")
+    feats = {}
+    rest = xpos[1:].ljust(8, "-")
+    person, number, tense, mood, voice, gender, case, degree = rest[:8]
+    if person in "123":
+        feats["Person"] = person
+    if number in _NUMBER:
+        feats["Number"] = _NUMBER[number]
+    if tense in _TENSE_ASPECT:
+        t, a = _TENSE_ASPECT[tense]
+        feats["Tense"] = t
+        if a:
+            feats["Aspect"] = a
+    if mood in _MOOD_VERBFORM:
+        m, vf = _MOOD_VERBFORM[mood]
+        if m:
+            feats["Mood"] = m
+        feats["VerbForm"] = vf
+    if voice in _VOICE:
+        feats["Voice"] = _VOICE[voice]
+    if gender in _GENDER:
+        feats["Gender"] = _GENDER[gender]
+    if case in _CASE:
+        feats["Case"] = _CASE[case]
+    if degree in _DEGREE:
+        feats["Degree"] = _DEGREE[degree]
+    return upos, feats
+
+
+# Straight per-label relabeling for everything that ISN'T COORD/AuxP/Pnom (those need the tree
+# restructuring in to_ud() below). Homoglyph artifacts in the training treebank (Greek "Ζ"/"Κ"
+# instead of Latin "Z"/"K" in a couple of AuxZ/AuxK instances) are normalized first.
+_HOMOGLYPH_FIX = {"AuxΖ": "AuxZ", "AuxΚ": "AuxK"}
+_SIMPLE_DEPREL = {
+    "SBJ": "nsubj", "OBJ": "obj", "OCOMP": "obj", "APOS": "appos",
+    "AuxC": "mark", "AuxV": "aux", "AuxG": "punct", "AuxK": "punct", "AuxX": "punct",
+    "AuxR": "expl", "AuxY": "cc", "AuxZ": "advmod", "AuxΖ": "advmod", "AuxΚ": "punct",
+    "ATV": "xcomp", "AtvV": "xcomp", "ExD": "orphan", "MWE": "fixed",
+    "PRED": "root",
+}
+
+
+def _normalize_deprel(label: str | None) -> str | None:
+    return _HOMOGLYPH_FIX.get(label, label)
+
+
+def to_ud(sent: list[dict]) -> list[dict]:
+    """Convert one sentence's native decode() output (list of {form, xpos, lemma, upos, head,
+    deprel} dicts, 1-indexed word order, head=0 meaning root) into UD-style output: each dict
+    becomes {form, lemma, upos (UD), xpos (native tag, kept as-is per UD convention that XPOS
+    is language-specific and optional), feats (dict), head, deprel (UD)}.
+
+    See the module-level comment above for the restructuring rules and scope/limitations.
+    """
+    n = len(sent)
+    if n == 0:
+        return []
+    heads = [0] * (n + 1)
+    deprels = [None] * (n + 1)
+    native_upos = [None] * (n + 1)
+    for i, w in enumerate(sent, start=1):
+        heads[i] = w["head"] if w["head"] is not None else 0
+        deprels[i] = _normalize_deprel(w["deprel"])
+        native_upos[i] = w["upos"]
+
+    def children_of():
+        c = {i: [] for i in range(0, n + 1)}
+        for i in range(1, n + 1):
+            c.setdefault(heads[i], []).append(i)
+        return c
+
+    # Stage 1: COORD -> first-conjunct-head restructuring.
+    for c in [i for i in range(1, n + 1) if deprels[i] == "COORD"]:
+        kids = sorted(children_of().get(c, []))
+        if not kids:
+            continue
+        first, rest = kids[0], kids[1:]
+        heads[first] = heads[c]
+        deprels[first] = "dep"   # COORD's own outbound label carries no recoverable function; see above
+        heads[c] = first
+        deprels[c] = "cc"
+        for k in rest:
+            heads[k] = first
+            if deprels[k] not in ("AuxG", "AuxK", "AuxX", "AuxΖ", "AuxΚ"):  # not a punctuation child
+                deprels[k] = "conj"
+            # punctuation children just reattach to `first`, keeping their own (later-relabeled) deprel
+
+    # Stage 2: AuxP -> case restructuring (nominal complement becomes the head).
+    for p in [i for i in range(1, n + 1) if deprels[i] == "AuxP"]:
+        kids = sorted(children_of().get(p, []))
+        if not kids:
+            continue
+        nominal, others = kids[0], kids[1:]
+        heads[nominal] = heads[p]
+        deprels[nominal] = "obl"   # AuxP's own outbound label carries no recoverable function; see above
+        heads[p] = nominal
+        deprels[p] = "case"
+        for o in others:
+            heads[o] = nominal
+
+    # Stage 3: Pnom (predicate nominal) -> copula reversal. The nominal becomes the head in
+    # place of the copula verb; the verb becomes `cop`; anything else attached to the verb
+    # (chiefly the subject) is reattached to the nominal.
+    for nom in [i for i in range(1, n + 1) if deprels[i] == "PNOM"]:
+        verb = heads[nom]
+        if verb == 0:
+            continue
+        heads[nom] = heads[verb]
+        deprels[nom] = deprels[verb] if deprels[verb] != "PRED" else "root"
+        heads[verb] = nom
+        deprels[verb] = "cop"
+        for i in range(1, n + 1):
+            if i != nom and heads[i] == verb:
+                heads[i] = nom
+
+    # Stage 4: everything else -- straight relabel, with a few UPOS/context-sensitive refinements.
+    for i in range(1, n + 1):
+        d = deprels[i]
+        if d in ("cc", "conj", "case", "cop", "dep", "obl"):     # already converted above
+            continue
+        if d == "ATR":
+            deprels[i] = ("det" if native_upos[i] == "l"
+                          else "amod" if native_upos[i] == "a"
+                          else "nmod")
+        elif d == "ADV":
+            deprels[i] = ("advmod" if native_upos[i] == "d"
+                          else "advcl" if native_upos[i] == "v" else "obl")
+        elif d in _SIMPLE_DEPREL:
+            deprels[i] = _SIMPLE_DEPREL[d]
+        # else: already UD-style (e.g. a second pass over an already-converted label) or
+        # unrecognized -- left as-is rather than silently dropped.
+
+    out = []
+    for i, w in enumerate(sent, start=1):
+        upos, feats = xpos_to_upos_feats(w["xpos"])
+        if deprels[i] == "mark" and upos == "CCONJ":
+            upos = "SCONJ"   # refine c (CCONJ default) once we know it introduced a subordinate clause
+        out.append(dict(form=w["form"], lemma=w["lemma"], upos=upos, xpos=w["xpos"],
+                         feats=feats, head=heads[i], deprel=deprels[i]))
+    return out
+
+
 # ---- edit-script machinery (tagger/edits.py, ported verbatim on strings) ---------------------
 
 GRAVE, ACUTE = "̀", "́"
@@ -355,10 +578,19 @@ class CharBertJointProcessor:
 
     # ---------------------------------------------------------------- full decode
 
-    def decode(self, model_out, batch, use_lexicon: bool = True) -> list[list[dict]]:
+    def decode(self, model_out, batch, use_lexicon: bool = True, ud: bool = False) -> list[list[dict]]:
         """model_out: a CharBertJointOutput (or the equivalent return_dict=False tuple).
         batch: the dict returned by __call__ (needs `_forms`).
-        -> per sentence, a list of {form, xpos, lemma, upos, head, deprel} dicts, word order."""
+        -> per sentence, a list of dicts, word order.
+
+        Default (ud=False): the model's NATIVE Perseus/AGDT-style output --
+        {form, xpos, lemma, upos, head, deprel}, where `upos` is the Perseus single-letter
+        code (not UD) and `deprel` is a Prague-style relation (SBJ/OBJ/ATR/COORD/AuxP/...).
+
+        ud=True: Universal Dependencies-style output instead -- {form, lemma, upos (UD),
+        xpos (native tag, kept as-is per UD convention), feats (dict), head, deprel (UD)}.
+        See `to_ud()` for the conversion (a documented, but not 100%-official-treebank-exact,
+        port of the HamleDT/UD_Ancient_Greek-Perseus conversion methodology)."""
         word_mask = model_out.word_mask if hasattr(model_out, "word_mask") else model_out[-1]
         xpos_logits = model_out.xpos_logits if hasattr(model_out, "xpos_logits") else model_out[0]
         script_logits = model_out.script_logits if hasattr(model_out, "script_logits") else model_out[1]
@@ -390,5 +622,5 @@ class CharBertJointProcessor:
                           if labels_out is not None and self.deprel_vocab else None)
                 sent_out.append(dict(form=form, xpos=xpos, lemma=lemma, upos=upos,
                                       head=head, deprel=deprel))
-            results.append(sent_out)
+            results.append(to_ud(sent_out) if ud else sent_out)
         return results