File size: 1,061 Bytes
bb36620 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | default_stage:
default_modifiers:
SpinQuantModifier:
transform_config:
config_groups:
R1:
type: hadamard
apply:
- targets: ['re:.*embed_tokens$', 're:.*o_proj$', 're:.*down_proj$']
location: weight_output
inverse: false
ignore: []
- targets: ['re:.*q_proj$', 're:.*k_proj$', 're:.*v_proj$', 're:.*up_proj$', 're:.*gate_proj$',
lm_head]
location: weight_input
inverse: true
ignore: []
randomize: false
requires_grad: false
head_dim: null
R2:
type: hadamard
apply:
- targets: ['re:.*v_proj$']
location: weight_output
inverse: false
ignore: []
- targets: ['re:.*o_proj$']
location: weight_input
inverse: true
ignore: []
randomize: false
requires_grad: false
head_dim: 128
|