aceOfDiamonds commited on
Commit
8e2b954
·
1 Parent(s): 4f1a7bb

Add a Vulkan fp16 variant

Browse files

fp16 is what makes this model worth lowering: at fp32 Vulkan loses to XNNPACK
on an Adreno 840 (0.91x), at fp16 it wins 1.35x, and on a Mali-G76 it is 1.90x.
Halving the file to 218 MB also eases the memory pressure that dominates this
model on 8 GB devices.

Mean pooling is written with keepdim so the reduction stays on the GPU, which
costs one delegate blob less. The 30527-row token embedding is kept off
texture storage, without which fp16 returns silently wrong values
(pytorch/executorch#22333); cosine against the CPU reference is 0.99995 on
Adreno and 0.99997 on Mali.

Requires the ExecuTorch 1.4.1 Vulkan runtime from the v0.10.0-libs-1.4.1
artifacts.

.gitattributes CHANGED
@@ -36,3 +36,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
36
  all-mpnet-base-v2.pte filter=lfs diff=lfs merge=lfs -text
37
  all-mpnet-base-v2_xnnpack.pte filter=lfs diff=lfs merge=lfs -text
38
  xnnpack/all_mpnet_base_v2_xnnpack_fp32.pte filter=lfs diff=lfs merge=lfs -text
 
 
36
  all-mpnet-base-v2.pte filter=lfs diff=lfs merge=lfs -text
37
  all-mpnet-base-v2_xnnpack.pte filter=lfs diff=lfs merge=lfs -text
38
  xnnpack/all_mpnet_base_v2_xnnpack_fp32.pte filter=lfs diff=lfs merge=lfs -text
39
+ vulkan/all_mpnet_base_v2_vulkan_fp16.pte filter=lfs diff=lfs merge=lfs -text
vulkan/all_mpnet_base_v2_vulkan_fp16.pte ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d891d3f6d3177d751edde4f4c9f436a05f7346addaf865a6ab8431830cf8b1a6
3
+ size 217946754
vulkan/config.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "$schema": "https://huggingface.co/software-mansion/react-native-executorch-spec/resolve/main/config.schema.json",
3
+ "model": "all_mpnet_base_v2",
4
+ "family": "sbert",
5
+ "capabilities": ["text-embedding"],
6
+ "backend": "vulkan",
7
+ "license": "apache-2.0",
8
+ "variants": [
9
+ {
10
+ "file": "all_mpnet_base_v2_vulkan_fp16.pte",
11
+ "precision": "fp16",
12
+ "quantized": false,
13
+ "default": true,
14
+ "methods": {
15
+ "forward": {
16
+ "inputs": [
17
+ { "shape": [1, 382], "dtype": "int64" },
18
+ { "shape": [1, 382], "dtype": "int64" }
19
+ ],
20
+ "outputs": [{ "shape": [1, 768], "dtype": "float32" }]
21
+ }
22
+ }
23
+ }
24
+ ]
25
+ }