Add verified Verdict Core ML L512 FP16

README.md

@@ -10,14 +10,14 @@ tags:
   - verdict
 ---
 
-# Verdict Core ML (L128 FP16)
+# Verdict Core ML (L128 and L512 FP16)
 
-This is a Core ML conversion of the **trained Verdict checkpoint** [`heman10x/rlcd-modernbert-151m`](https://huggingface.co/heman10x/rlcd-modernbert-151m), pinned at `8af2496eb63c7fa66d7d234e1f62629380030eb4`. It has 151,378,177 parameters and retains the full encoder plus trained decision head. The `verdict_fp16_L128_candidates25.mlpackage` file is 303,210,832 bytes and targets iOS 17 / macOS 14 or newer.
+This is a Core ML conversion of the **trained Verdict checkpoint** [`heman10x/rlcd-modernbert-151m`](https://huggingface.co/heman10x/rlcd-modernbert-151m), pinned at `8af2496eb63c7fa66d7d234e1f62629380030eb4`. It has 151,378,177 parameters and retains the full encoder plus trained decision head. The L128 package is 303,210,832 bytes; L512 is 304,390,482 bytes. Both target iOS 17 / macOS 14 or newer. Choose L128 when the full rendered request fits 128 tokens and L512 for 129–512 tokens. Do not truncate an overlength request into an installed bucket.
 
-Inputs are `input_ids` and `attention_mask` of shape `[1,128]`, and `class_marker_map` of shape `[1,25,128]`. The graph returns raw logits and an uncalibrated softmax. For a native Verdict decision, render the request with `native_reference.py`, append the `__insufficient_evidence__` candidate, and apply `calibrator.json` to raw logits. This means at most **24 substantive candidates**. The released calibrator has a separate temperature for several candidate counts. Calling the graph with arbitrary labels or treating `probabilities` as calibrated changes the model's behavior.
+Inputs are `input_ids` and `attention_mask` of shape `[1,L]`, and `class_marker_map` of shape `[1,25,L]`, where `L` is 128 or 512. The graph returns raw logits and an uncalibrated softmax. For a native Verdict decision, render the request with `native_reference.py`, append the `__insufficient_evidence__` candidate, and apply `calibrator.json` to raw logits. This means at most **24 substantive candidates**. The released calibrator has a separate temperature for several candidate counts. Calling the graph with arbitrary labels or treating `probabilities` as calibrated changes the model's behavior.
 
-Verified on Apple M5 Pro/macOS 27.0 against the pinned PyTorch model: 3/3 choice, noul and score smoke decisions agreed, with worst calibrated probability difference 0.00341. The 20-iteration `coreml-cli` median was 3.627 ms CPU+GPU, 3.710 ms CPU+ANE, 3.915 ms automatic, and 11.721 ms CPU only. These are model call times; rendering and tokenization add time. The `verification-L128.json` and `profile-L128.json` files include the details.
+Verified on Apple M5 Pro/macOS 27.0 against the pinned PyTorch model: L128 passed 4/4 smoke decisions (including two native abstentions), worst calibrated probability difference 0.00341. L512 passed 5/5 (including one 283-token public Decision Index row), worst difference 0.00070. The 20-iteration L128 `coreml-cli` median was 3.627 ms CPU+GPU, 3.710 ms CPU+ANE, 3.915 ms automatic, and 11.721 ms CPU only. L512 median automatic model call time across its five parity requests was 8.15 ms. Rendering and tokenization add time. The verification and profile JSON files include the details.
 
-The public Decision Index tracker reports Verdict at 13.38. Its historical checkpoint and renderer have not been authenticated against this pinned release, so this artifact does **not** claim to reproduce that score. Only the L128 bucket has been verified; larger inputs require a separately exported and tested bucket. Unsupported requests must not be truncated to 128 tokens without stating that behavior.
+The public Decision Index tracker reports Verdict at 13.38. Its historical checkpoint and renderer have not been authenticated against this pinned release, so this artifact does **not** claim to reproduce that score. The two buckets cover its 512-token public limit, but full-suite correctness and quantized parity are still pending.
 
 The Hub repo includes the runtime renderer, calibration helper, tokenizer, verification report and SHA-256 asset lock. Conversion source is maintained in [FluidInference/mobius](https://github.com/FluidInference/mobius). Upstream model and code: [Verdict model](https://huggingface.co/heman10x/rlcd-modernbert-151m), [Verdict source](https://github.com/Heman10x-NGU/Verdict-open-jev), Apache-2.0. Base architecture: [`knowledgator/gliclass-modern-base-v2.0`](https://huggingface.co/knowledgator/gliclass-modern-base-v2.0).

native_reference.py

@@ -62,6 +62,8 @@ def build_request(context: str, question: dict[str, Any]) -> Request:
         raise ValueError(f"unknown Verdict question type: {kind}")
     labels += (ABSTAIN_DESCRIPTION,)
     ids += (ABSTAIN_ID,)
+    if len(ids) != len(set(ids)):
+        raise ValueError("Verdict candidate IDs must be unique and cannot use the abstention ID")
     prefix = "".join(f"<<LABEL>>{label}" for label in labels)
     return Request(kind, f"{prefix}<<SEP>>{text}", labels, ids, values)
 
@@ -86,10 +88,14 @@ def decode(logits: np.ndarray, request: Request, calibrator: dict[str, Any]) ->
     distribution = dict(zip(request.ids, map(float, p)))
     selected_id = request.ids[int(p.argmax())]
     abstained = selected_id == ABSTAIN_ID
+    entropy = -sum(float(value) * math.log(float(value)) for value in p if value > 1e-12)
+    concentration = max(0.0, min(1.0, 1.0 - entropy / math.log(k))) if k > 1 else 1.0
     result: dict[str, Any] = {
         "type": request.kind,
         "selected_id": selected_id,
+        "selected_probability": distribution[selected_id],
         "probabilities": distribution,
+        "concentration": concentration,
         "is_abstention": abstained,
         "p_abstain": distribution[ABSTAIN_ID],
         "calibration_status": "calibrated_for_scope",
@@ -99,6 +105,7 @@ def decode(logits: np.ndarray, request: Request, calibrator: dict[str, Any]) ->
     elif request.kind == "noul":
         substantive = distribution["true"] + distribution["false"]
         result["noul"] = distribution["true"] / substantive if substantive and not abstained else None
+        result["selected_outcome"] = selected_id
     else:
         substantive = sum(distribution[key] for key in request.ids[:-1])
         result["score"] = (
@@ -106,6 +113,10 @@ def decode(logits: np.ndarray, request: Request, calibrator: dict[str, Any]) ->
             if substantive and not abstained
             else None
         )
+        result["selected_level_id"] = selected_id
+        result["selected_value"] = (
+            request.values[request.ids.index(selected_id)] if not abstained else None
+        )
     return result
 
 

preprocessing.py

@@ -5,13 +5,15 @@ from __future__ import annotations
 import numpy as np
 
 
-def prepare(tokenizer, model, rendered: str, length: int, max_candidates: int) -> dict[str, np.ndarray]:
+def prepare(
+    tokenizer, class_token_index: int, rendered: str, length: int, max_candidates: int
+) -> dict[str, np.ndarray]:
     full = tokenizer(rendered, truncation=False)
     if len(full["input_ids"]) > length:
         raise ValueError(f"Verdict prompt needs {len(full['input_ids'])} tokens; L{length} has no room")
     encoded = tokenizer(rendered, truncation=False, padding="max_length", max_length=length, return_tensors="np")
     ids = encoded["input_ids"].astype(np.int32)
-    positions = np.flatnonzero(ids[0] == model.config.class_token_index)
+    positions = np.flatnonzero(ids[0] == class_token_index)
     if len(positions) > max_candidates:
         raise ValueError("candidate markers exceed exported head capacity")
     markers = np.zeros((1, max_candidates, length), dtype=np.float32)

verdict_fp16_L512_candidates25.conversion.json

@@ -0,0 +1,15 @@
+{
+  "package": "verdict_fp16_L512_candidates25.mlpackage",
+  "source_repo": "heman10x/rlcd-modernbert-151m",
+  "source_revision": "8af2496eb63c7fa66d7d234e1f62629380030eb4",
+  "source_weight_sha256": "d252823994d47a7933217fc86449493299643af6a0c0d83d6bd5a7666d3253ef",
+  "parameters": 151378177,
+  "wrapper_max_logit_error": 1.9073486328125e-06,
+  "conversion_seconds": 5.1714104160200804,
+  "package_bytes": 304390482,
+  "package_files_sha256": {
+    "Manifest.json": "047cc80f7d66d822ca0d9c0100a6a6dc96ae2d70eac42e292a2b246e01e64ddd",
+    "Data/com.apple.CoreML/model.mlmodel": "7ad21a9ee93d1c573e6d89e6c35511f08c984a5c47108bb2b908c96344c03903",
+    "Data/com.apple.CoreML/weights/weight.bin": "fb438de5823dd91c8448b373d026b7e6dcb7149bf871fde54057fd6d82586886"
+  }
+}

verdict_fp16_L512_candidates25.mlpackage/Data/com.apple.CoreML/model.mlmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ad21a9ee93d1c573e6d89e6c35511f08c984a5c47108bb2b908c96344c03903
+size 306985

verdict_fp16_L512_candidates25.mlpackage/Data/com.apple.CoreML/weights/weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb438de5823dd91c8448b373d026b7e6dcb7149bf871fde54057fd6d82586886
+size 304082880

verdict_fp16_L512_candidates25.mlpackage/Manifest.json

@@ -0,0 +1,18 @@
+{
+    "fileFormatVersion": "1.0.0",
+    "itemInfoEntries": {
+        "8DCAA6D5-D8F6-4FC5-B35A-D8D1776F73F2": {
+            "author": "com.apple.CoreML",
+            "description": "CoreML Model Specification",
+            "name": "model.mlmodel",
+            "path": "com.apple.CoreML/model.mlmodel"
+        },
+        "9CE3A02A-DD4B-448B-AA54-2D34CD87C941": {
+            "author": "com.apple.CoreML",
+            "description": "CoreML Model Weights",
+            "name": "weights",
+            "path": "com.apple.CoreML/weights"
+        }
+    },
+    "rootModelIdentifier": "8DCAA6D5-D8F6-4FC5-B35A-D8D1776F73F2"
+}

verification-L512.json

@@ -0,0 +1,216 @@
+{
+  "source_repo": "heman10x/rlcd-modernbert-151m",
+  "source_revision": "8af2496eb63c7fa66d7d234e1f62629380030eb4",
+  "package": "verdict_fp16_L512_candidates25.mlpackage",
+  "chip": "arm",
+  "macos": "27.0",
+  "max_calibrated_probability_error": 0.0006975581321209678,
+  "selected_id_agreements": 5,
+  "questions": 5,
+  "rows": [
+    {
+      "name": "lost card",
+      "type": "choice",
+      "tokens": 53,
+      "candidates_including_abstention": 3,
+      "native": {
+        "type": "choice",
+        "selected_id": "card_lost",
+        "selected_probability": 0.6279954438939314,
+        "probabilities": {
+          "card_lost": 0.6279954438939314,
+          "pin_reset": 0.12699384187218735,
+          "__insufficient_evidence__": 0.2450107142338813
+        },
+        "concentration": 0.18185852825435922,
+        "is_abstention": false,
+        "p_abstain": 0.2450107142338813,
+        "calibration_status": "calibrated_for_scope",
+        "choice": "card_lost"
+      },
+      "coreml": {
+        "type": "choice",
+        "selected_id": "card_lost",
+        "selected_probability": 0.6286024758124755,
+        "probabilities": {
+          "card_lost": 0.6286024758124755,
+          "pin_reset": 0.12674703377209953,
+          "__insufficient_evidence__": 0.24465049041542497
+        },
+        "concentration": 0.18252696123986079,
+        "is_abstention": false,
+        "p_abstain": 0.24465049041542497,
+        "calibration_status": "calibrated_for_scope",
+        "choice": "card_lost"
+      },
+      "max_calibrated_probability_error": 0.0006070319185441653,
+      "selected_id_agrees": true,
+      "median_model_call_ms": 8.042125002248213
+    },
+    {
+      "name": "binary inquiry",
+      "type": "noul",
+      "tokens": 50,
+      "candidates_including_abstention": 3,
+      "native": {
+        "type": "noul",
+        "selected_id": "true",
+        "selected_probability": 0.4692844463560151,
+        "probabilities": {
+          "true": 0.4692844463560151,
+          "false": 0.24040823180168494,
+          "__insufficient_evidence__": 0.29030732184229996
+        },
+        "concentration": 0.03808302501474403,
+        "is_abstention": false,
+        "p_abstain": 0.29030732184229996,
+        "calibration_status": "calibrated_for_scope",
+        "noul": 0.6612502295701238,
+        "selected_outcome": "true"
+      },
+      "coreml": {
+        "type": "noul",
+        "selected_id": "true",
+        "selected_probability": 0.46998200448813604,
+        "probabilities": {
+          "true": 0.46998200448813604,
+          "false": 0.23993786932412314,
+          "__insufficient_evidence__": 0.29008012618774076
+        },
+        "concentration": 0.03846968937399986,
+        "is_abstention": false,
+        "p_abstain": 0.29008012618774076,
+        "calibration_status": "calibrated_for_scope",
+        "noul": 0.6620211967927305,
+        "selected_outcome": "true"
+      },
+      "max_calibrated_probability_error": 0.0006975581321209678,
+      "selected_id_agrees": true,
+      "median_model_call_ms": 8.013729006052017
+    },
+    {
+      "name": "urgency score",
+      "type": "score",
+      "tokens": 55,
+      "candidates_including_abstention": 4,
+      "native": {
+        "type": "score",
+        "selected_id": "__insufficient_evidence__",
+        "selected_probability": 0.3259148157325998,
+        "probabilities": {
+          "low": 0.19793214504236917,
+          "medium": 0.2898236019444062,
+          "high": 0.18632943728062487,
+          "__insufficient_evidence__": 0.3259148157325998
+        },
+        "concentration": 0.02039165010923949,
+        "is_abstention": true,
+        "p_abstain": 0.3259148157325998,
+        "calibration_status": "calibrated_for_scope",
+        "score": null,
+        "selected_level_id": "__insufficient_evidence__",
+        "selected_value": null
+      },
+      "coreml": {
+        "type": "score",
+        "selected_id": "__insufficient_evidence__",
+        "selected_probability": 0.3262236353210236,
+        "probabilities": {
+          "low": 0.19780841262370594,
+          "medium": 0.28950115689828004,
+          "high": 0.18646679515699038,
+          "__insufficient_evidence__": 0.3262236353210236
+        },
+        "concentration": 0.020408360566200767,
+        "is_abstention": true,
+        "p_abstain": 0.3262236353210236,
+        "calibration_status": "calibrated_for_scope",
+        "score": null,
+        "selected_level_id": "__insufficient_evidence__",
+        "selected_value": null
+      },
+      "max_calibrated_probability_error": 0.000322445046126163,
+      "selected_id_agrees": true,
+      "median_model_call_ms": 8.470937464153394
+    },
+    {
+      "name": "insufficient evidence",
+      "type": "choice",
+      "tokens": 41,
+      "candidates_including_abstention": 3,
+      "native": {
+        "type": "choice",
+        "selected_id": "__insufficient_evidence__",
+        "selected_probability": 0.6003396635578521,
+        "probabilities": {
+          "visa": 0.19581437244279193,
+          "mastercard": 0.2038459639993559,
+          "__insufficient_evidence__": 0.6003396635578521
+        },
+        "concentration": 0.1354398243565177,
+        "is_abstention": true,
+        "p_abstain": 0.6003396635578521,
+        "calibration_status": "calibrated_for_scope",
+        "choice": "__insufficient_evidence__"
+      },
+      "coreml": {
+        "type": "choice",
+        "selected_id": "__insufficient_evidence__",
+        "selected_probability": 0.6008594846026971,
+        "probabilities": {
+          "visa": 0.19567826624893642,
+          "mastercard": 0.2034622491483665,
+          "__insufficient_evidence__": 0.6008594846026971
+        },
+        "concentration": 0.13595645758378905,
+        "is_abstention": true,
+        "p_abstain": 0.6008594846026971,
+        "calibration_status": "calibrated_for_scope",
+        "choice": "__insufficient_evidence__"
+      },
+      "max_calibrated_probability_error": 0.000519821044845048,
+      "selected_id_agrees": true,
+      "median_model_call_ms": 8.155353978509083
+    },
+    {
+      "name": "BFCL-tool-selection:simple:simple_254",
+      "type": "choice",
+      "tokens": 283,
+      "candidates_including_abstention": 3,
+      "native": {
+        "type": "choice",
+        "selected_id": "yes",
+        "selected_probability": 0.5095718104586857,
+        "probabilities": {
+          "yes": 0.5095718104586857,
+          "no": 0.28447377613128705,
+          "__insufficient_evidence__": 0.20595441341002724
+        },
+        "concentration": 0.0655574261710149,
+        "is_abstention": false,
+        "p_abstain": 0.20595441341002724,
+        "calibration_status": "calibrated_for_scope",
+        "choice": "yes"
+      },
+      "coreml": {
+        "type": "choice",
+        "selected_id": "yes",
+        "selected_probability": 0.5093345700124705,
+        "probabilities": {
+          "yes": 0.5093345700124705,
+          "no": 0.28465867339056805,
+          "__insufficient_evidence__": 0.20600675659696147
+        },
+        "concentration": 0.06541626747983909,
+        "is_abstention": false,
+        "p_abstain": 0.20600675659696147,
+        "calibration_status": "calibrated_for_scope",
+        "choice": "yes"
+      },
+      "max_calibrated_probability_error": 0.00023724044621520335,
+      "selected_id_agrees": true,
+      "median_model_call_ms": 8.165458013536409
+    }
+  ],
+  "passed": true
+}