dagloop5 commited on
Commit
6c70c31
·
verified ·
1 Parent(s): 5df3dca

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +12 -3
app.py CHANGED
@@ -634,9 +634,18 @@ def _generate(
634
  # to call on every request. Filtered to `LOADED_LORAS`: a slider for a LoRA that failed at startup has no
635
  # adapter behind it, and `set_adapters` would raise if asked to activate a name that was never attached.
636
  if LOADED_LORAS:
637
- active = {name: strength for name, strength in lora_strengths.items() if name in LOADED_LORAS}
638
- if active:
639
- PIPE.transformer.set_adapters(list(active), weights=list(active.values()))
 
 
 
 
 
 
 
 
 
640
 
641
  if PLACEMENT == "lazy":
642
  PIPE.to("cuda")
 
634
  # to call on every request. Filtered to `LOADED_LORAS`: a slider for a LoRA that failed at startup has no
635
  # adapter behind it, and `set_adapters` would raise if asked to activate a name that was never attached.
636
  if LOADED_LORAS:
637
+ # Filtered to strength > 0, not just "loaded": PEFT computes every adapter in the active list on every
638
+ # forward regardless of its weight (no early-exit for scale 0), so an adapter left active at 0.0 still
639
+ # costs a real lora_A/lora_B matmul per targeted Linear, every block, every step — overhead that scales
640
+ # with how many LoRAs are loaded, not how many are actually in use for a given request. Called
641
+ # unconditionally, even with an empty list, rather than only `if active:` — skipping the call when every
642
+ # slider is 0 would leave whichever adapters the *previous* request activated still live.
643
+ active = {
644
+ name: strength
645
+ for name, strength in lora_strengths.items()
646
+ if name in LOADED_LORAS and strength > 0
647
+ }
648
+ PIPE.transformer.set_adapters(list(active), weights=list(active.values()))
649
 
650
  if PLACEMENT == "lazy":
651
  PIPE.to("cuda")