Hugging Face's logo

FluidInference
/
laya-coreml

Text Classification
Core ML
laya
apple-silicon
neural-engine
decision-model
fluidaudio
Model card Files
xet
 Community
laya-coreml
File size: 2,954 Bytes
40e4036
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
{
  "protocol": "Reproduce the five-task \"Laya vs Jev, measured\" table (brainfunctioncollapse.com/laya) on device.\n\nThe post gives task names, 100 labelled examples per task, laya 0.3.4 English checkpoint on an M1 Max\nGPU, and Jev's accuracy, but not its datasets, sampling or question wording. This script uses the\nobvious public dataset for each task, the first 100 rows of its test split (train for SMS spam, which\nhas no test split), and laya's own question wording where one exists. Jev is a closed API and is not\nmeasured here; its column is copied from the post. Both laya checkpoints are scored with the\nunmodified PyTorch runtime so the English numbers can be compared with the post and the\nmultilingual rows give the reference for the Core ML buckets (`FluidUseLaya benchmark`).\n\nOutputs:\n  benchmark/jev-suites.jsonl           the 500 questions with serialized states and gold labels\n  benchmark/jev-reference-rows.jsonl   multilingual PyTorch answers per row\n  reports/benchmark-jev-reference.json accuracy per task for both checkpoints",
  "n_per_task": 100,
  "post": {
    "news_topic": {
      "laya_english_m1max": 0.93,
      "jev": 0.92
    },
    "sms_spam": {
      "laya_english_m1max": 0.96,
      "jev": 0.96
    },
    "emotion": {
      "laya_english_m1max": 0.45,
      "jev": 0.53
    },
    "review_stars": {
      "laya_english_m1max": 0.35,
      "jev": 0.7
    },
    "prompt_injection": {
      "laya_english_m1max": 0.65,
      "jev": 0.71
    },
    "all": {
      "laya_english_m1max": 0.668,
      "jev": 0.764
    }
  },
  "checkpoints": {
    "multilingual": {
      "max_len": 1024,
      "seconds": 21.3,
      "ms_per_question": 42.55,
      "emotion": {
        "n": 100,
        "accuracy": 0.58,
        "max_tokens": 82
      },
      "news_topic": {
        "n": 100,
        "accuracy": 0.97,
        "max_tokens": 226
      },
      "prompt_injection": {
        "n": 100,
        "accuracy": 0.64,
        "max_tokens": 177
      },
      "review_stars": {
        "n": 100,
        "accuracy": 0.35,
        "max_tokens": 782
      },
      "sms_spam": {
        "n": 100,
        "accuracy": 0.58,
        "max_tokens": 118
      },
      "all": {
        "n": 500,
        "accuracy": 0.624
      }
    },
    "english": {
      "max_len": 512,
      "seconds": 48.9,
      "ms_per_question": 97.87,
      "emotion": {
        "n": 100,
        "accuracy": 0.65,
        "max_tokens": 83
      },
      "news_topic": {
        "n": 100,
        "accuracy": 0.95,
        "max_tokens": 232
      },
      "prompt_injection": {
        "n": 100,
        "accuracy": 0.79,
        "max_tokens": 252
      },
      "review_stars": {
        "n": 100,
        "accuracy": 0.39,
        "max_tokens": 512
      },
      "sms_spam": {
        "n": 100,
        "accuracy": 0.88,
        "max_tokens": 116
      },
      "all": {
        "n": 500,
        "accuracy": 0.732
      }
    }
  }
}