Add calibrated thresholds (IdeaLens-1M calibration split)

thresholds.json

@@ -0,0 +1,185 @@
+{
+ "model": "rishanthrajendhran/IdeaLens-LogisticClassifier-PerItem",
+ "input": "outline items, pooled per document by logit mean",
+ "flag_rule": "flag the document as AI when P(human) < cut",
+ "calibration": {
+  "data": "IdeaLens-1M, calibration split (human documents only)",
+  "scores_from": "the released coefficients applied to the stored text-embedding-3-large item vectors",
+  "n_humans": 79999,
+  "n_per_format": {
+   "Academic Writing": 9999,
+   "Creative Writing": 10000,
+   "Knowledge Article": 10000,
+   "News Article": 10000,
+   "Nonfiction Writing": 10000,
+   "Personal About Page": 10000,
+   "Personal Blog": 10000,
+   "User Reviews": 10000
+  },
+  "n_per_topic": {
+   "Art & Design": 3338,
+   "Crime & Law": 3931,
+   "Education & Jobs": 3974,
+   "Entertainment": 4692,
+   "Fashion & Beauty": 2999,
+   "Finance & Business": 4855,
+   "Food & Dining": 2696,
+   "Games": 4257,
+   "Hardware": 2253,
+   "Health": 5068,
+   "History": 2554,
+   "Home & Hobbies": 4192,
+   "Industrial": 2299,
+   "Literature": 3571,
+   "Politics": 3728,
+   "Religion": 3617,
+   "Science & Tech.": 4154,
+   "Social Life": 3126,
+   "Software": 2257,
+   "Software Dev.": 2468,
+   "Sports & Fitness": 4224,
+   "Transportation": 2810,
+   "Travel": 2936
+  }
+ },
+ "method": {
+  "global": "the target-FPR quantile of P(human) over all calibration humans",
+  "per_group": "the group's own quantile, shrunk toward the global cut with weight n / (n + 2500)",
+  "min_group_humans": 200,
+  "estimability": "a cut at FPR q is given only when q * n >= 25 calibration humans",
+  "missing_cut": "no cut for a group means none could be estimated; do not substitute the global cut"
+ },
+ "fpr_targets": [
+  0.001,
+  0.005,
+  0.01,
+  0.02,
+  0.05
+ ],
+ "global": {
+  "0.001": 0.106388772270725,
+  "0.005": 0.1681378592028679,
+  "0.01": 0.20413262820741293,
+  "0.02": 0.25004924623511837,
+  "0.05": 0.3322128812111102
+ },
+ "per_format": {
+  "0.005": {
+   "Academic Writing": 0.18337902910718276,
+   "Creative Writing": 0.30439763961606603,
+   "Knowledge Article": 0.14825887050992825,
+   "News Article": 0.21597015644270443,
+   "Nonfiction Writing": 0.11824042645135695,
+   "Personal About Page": 0.27491334818253615,
+   "Personal Blog": 0.1835801423738712,
+   "User Reviews": 0.1822318018901514
+  },
+  "0.01": {
+   "Academic Writing": 0.21607582239079004,
+   "Creative Writing": 0.3518106575501341,
+   "Knowledge Article": 0.17779299087492645,
+   "News Article": 0.2525656340605388,
+   "Nonfiction Writing": 0.14773132496238617,
+   "Personal About Page": 0.31017104459922873,
+   "Personal Blog": 0.21251108111321929,
+   "User Reviews": 0.2098093606513001
+  },
+  "0.02": {
+   "Academic Writing": 0.2554967834761668,
+   "Creative Writing": 0.3997204895204556,
+   "Knowledge Article": 0.2120910185687447,
+   "News Article": 0.2990235999808508,
+   "Nonfiction Writing": 0.18482007099351194,
+   "Personal About Page": 0.3564756679939256,
+   "Personal Blog": 0.261089641004104,
+   "User Reviews": 0.24904559847959068
+  },
+  "0.05": {
+   "Academic Writing": 0.32697615729587837,
+   "Creative Writing": 0.48468336926562566,
+   "Knowledge Article": 0.28104233592914357,
+   "News Article": 0.3753134671745085,
+   "Nonfiction Writing": 0.26154657668290016,
+   "Personal About Page": 0.4343046934734537,
+   "Personal Blog": 0.3443488804918288,
+   "User Reviews": 0.3197188287873698
+  }
+ },
+ "per_topic": {
+  "0.005": {
+   "Health": 0.15962406372266164
+  },
+  "0.01": {
+   "Art & Design": 0.2211545545503707,
+   "Crime & Law": 0.25305002355029654,
+   "Education & Jobs": 0.2139546656223252,
+   "Entertainment": 0.20914571654184425,
+   "Fashion & Beauty": 0.19762709891151295,
+   "Finance & Business": 0.18093483800959098,
+   "Food & Dining": 0.204809748335236,
+   "Games": 0.1667155824543731,
+   "Health": 0.19673357032062658,
+   "History": 0.25187748492855894,
+   "Home & Hobbies": 0.20447598807553452,
+   "Literature": 0.22916258254951682,
+   "Politics": 0.24911769963488406,
+   "Religion": 0.20919173178201625,
+   "Science & Tech.": 0.20377505068961643,
+   "Social Life": 0.19915882208418206,
+   "Sports & Fitness": 0.22463184358387936,
+   "Transportation": 0.21942885157695088,
+   "Travel": 0.20809697694219897
+  },
+  "0.02": {
+   "Art & Design": 0.2762469725786032,
+   "Crime & Law": 0.29662344711917477,
+   "Education & Jobs": 0.2524994672382303,
+   "Entertainment": 0.2636351347996365,
+   "Fashion & Beauty": 0.2343747318477249,
+   "Finance & Business": 0.22362090442878566,
+   "Food & Dining": 0.25030645924418576,
+   "Games": 0.20406433258666584,
+   "Hardware": 0.26096374192469013,
+   "Health": 0.23460664586144925,
+   "History": 0.29367418646928634,
+   "Home & Hobbies": 0.24609285218609128,
+   "Industrial": 0.2542443043330692,
+   "Literature": 0.28009436211038863,
+   "Politics": 0.29651950168136987,
+   "Religion": 0.26923767380323155,
+   "Science & Tech.": 0.2637455277406645,
+   "Social Life": 0.24080888051854882,
+   "Software": 0.2299233322225545,
+   "Software Dev.": 0.2641363031676164,
+   "Sports & Fitness": 0.26836760830718515,
+   "Transportation": 0.27017252682668125,
+   "Travel": 0.2580344281855586
+  },
+  "0.05": {
+   "Art & Design": 0.3754907901312724,
+   "Crime & Law": 0.38690683761124794,
+   "Education & Jobs": 0.331575942940869,
+   "Entertainment": 0.3564682836223585,
+   "Fashion & Beauty": 0.31063629773891316,
+   "Finance & Business": 0.29528534633717574,
+   "Food & Dining": 0.3214608348894217,
+   "Games": 0.27430532429218063,
+   "Hardware": 0.32495907494838105,
+   "Health": 0.31631279315037686,
+   "History": 0.3751332954192705,
+   "Home & Hobbies": 0.33077720862652954,
+   "Industrial": 0.33217862714123064,
+   "Literature": 0.359404867142336,
+   "Politics": 0.3779864562921655,
+   "Religion": 0.35723814128703396,
+   "Science & Tech.": 0.3562150315640888,
+   "Social Life": 0.3119854343138797,
+   "Software": 0.302595137571614,
+   "Software Dev.": 0.33573465662440605,
+   "Sports & Fitness": 0.35078666506255834,
+   "Transportation": 0.3498444893449201,
+   "Travel": 0.33955383116682636
+  }
+ },
+ "pooling": "logit_mean"
+}