| """What the readout says about its own answer, with nothing fitted behind it. |
| |
| A question is decoded correctly when every unit of it is, so the readout's statement about a question is |
| the product of its statement about each unit. No parameter is fitted here; the only parameter on this path |
| is the per-task temperature in `solomon/calibration.py`. |
| |
| The product assumes the units of a question are independent. That has never been validated as a joint |
| probability: it orders multi-candidate questions well and it is not a calibrated joint. Read the |
| per-candidate numbers if you need a magnitude. `solomon.service.ORDERING_DISCLOSURE` says the same thing |
| in the response itself. |
| """ |
| import numpy as np |
| from scipy.special import logsumexp |
|
|
| TASKS = ('boolean', 'single', 'ordered', 'multilabel', 'entity') |
|
|
|
|
| def unit_correct_probability(unit, t=1.0): |
| """The readout's stated P(this unit is decoded correctly). |
| |
| Noul -> max(p, 1-p) with p = P(yes); the prediction is argmax, so this is P(prediction right). |
| Choice -> the listed top-1 probability. |
| t rescales the logits; t = 1 is the raw readout. |
| """ |
| if unit['kind'] == 'noul': |
| x = np.asarray([unit['z'], 0.0], float) / t |
| else: |
| x = np.asarray(unit['logits'], float) / t |
| return float(np.exp(x.max() - logsumexp(x))) |
|
|
|
|
| def question_reliability(units, t=1.0): |
| """P(every unit correct) under the readout, assuming unit independence within a question.""" |
| return float(np.prod([unit_correct_probability(u, t) for u in units])) |
|
|
