minhahwang Copilot commited on
Commit
b714046
·
1 Parent(s): bea718a

fix: audio decode normalization + warmup first 5 paragraphs

Browse files

- inference_lfm.py: normalize waveform to [-1,1], handle dtype/device issues,
detect garbage audio (too quiet), proper float32 conversion
- qa_flow.py: write WAV as FLOAT subtype for quality
- app.py: warmup expanded from 1 to 5 paragraphs per story

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Files changed (4) hide show
  1. app.py +11 -6
  2. assets/vivian_reference.wav +3 -0
  3. inference_lfm.py +23 -5
  4. qa_flow.py +4 -1
app.py CHANGED
@@ -423,28 +423,33 @@ print("[MomsVoice] LFM Q&A model ready. All models preloaded.")
423
  import threading
424
 
425
  def _warmup_first_paragraphs():
426
- """Background: pre-generate first paragraph of each story with stock voice to disk cache."""
427
  import logging as _log
428
  from tts import _get_cached_audio, _save_cached_audio, _synthesize_single
429
  _logger = _log.getLogger("warmup")
 
430
  for book in mock_books:
431
  try:
432
  paras = load_paragraphs(book["story_path"])
433
  if not paras:
434
  continue
435
- first_chunks = split_into_chunks(paras[0])
436
- for chunk in first_chunks:
 
 
 
437
  if _get_cached_audio(chunk, None) is not None:
438
  continue # Already cached
439
  wav, sr = _synthesize_single(chunk, None)
440
  _save_cached_audio(chunk, None, wav, sr)
441
- _logger.info("Warmed: %s (%d chunks)", book["title"], len(first_chunks))
 
442
  except Exception as e:
443
  _logger.warning("Warmup failed for %s: %s", book.get("title", "?"), e)
444
- _logger.info("All first paragraphs warmed.")
445
 
446
  threading.Thread(target=_warmup_first_paragraphs, daemon=True).start()
447
- print("[MomsVoice] Background warmup started: pre-generating first paragraph for all stories.")
448
 
449
 
450
  # Gradio Application Core setup
 
423
  import threading
424
 
425
  def _warmup_first_paragraphs():
426
+ """Background: pre-generate first 5 paragraphs of each story with stock voice to disk cache."""
427
  import logging as _log
428
  from tts import _get_cached_audio, _save_cached_audio, _synthesize_single
429
  _logger = _log.getLogger("warmup")
430
+ WARMUP_PARAS = 5
431
  for book in mock_books:
432
  try:
433
  paras = load_paragraphs(book["story_path"])
434
  if not paras:
435
  continue
436
+ # Pre-generate first 5 paragraphs
437
+ target_paras = paras[:WARMUP_PARAS]
438
+ all_chunks = split_into_chunks("\n\n".join(target_paras))
439
+ generated = 0
440
+ for chunk in all_chunks:
441
  if _get_cached_audio(chunk, None) is not None:
442
  continue # Already cached
443
  wav, sr = _synthesize_single(chunk, None)
444
  _save_cached_audio(chunk, None, wav, sr)
445
+ generated += 1
446
+ _logger.info("Warmed: %s (%d chunks, %d new)", book["title"], len(all_chunks), generated)
447
  except Exception as e:
448
  _logger.warning("Warmup failed for %s: %s", book.get("title", "?"), e)
449
+ _logger.info("All stories warmed (first %d paragraphs each).", WARMUP_PARAS)
450
 
451
  threading.Thread(target=_warmup_first_paragraphs, daemon=True).start()
452
+ print("[MomsVoice] Background warmup started: pre-generating first 5 paragraphs for all stories.")
453
 
454
 
455
  # Gradio Application Core setup
assets/vivian_reference.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ddd916e813a290c83be9aad1839f58739be153749d47a3e01d926b1cde26bf4e
3
+ size 368684
inference_lfm.py CHANGED
@@ -152,10 +152,28 @@ def answer_question_audio(
152
  # Decode audio
153
  waveform = None
154
  if audio_out and len(audio_out) > 1:
155
- decode_device = _module_device(processor, _module_device(model, _select_device()))
156
- audio_codes = torch.stack(audio_out[:-1], 1).unsqueeze(0).to(decode_device)
157
- with GPU_INFERENCE_LOCK, torch.inference_mode():
158
- waveform_tensor = processor.decode(audio_codes)
159
- waveform = waveform_tensor.cpu().numpy().squeeze()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
160
 
161
  return answer_text, waveform, SAMPLE_RATE
 
152
  # Decode audio
153
  waveform = None
154
  if audio_out and len(audio_out) > 1:
155
+ try:
156
+ decode_device = _module_device(processor, _module_device(model, _select_device()))
157
+ # Stack all audio codes except the final EOS marker
158
+ audio_codes = torch.stack(audio_out[:-1], 1).unsqueeze(0)
159
+ # Ensure codes are on the right device and in int/long format
160
+ if audio_codes.is_floating_point():
161
+ audio_codes = audio_codes.long()
162
+ audio_codes = audio_codes.to(decode_device)
163
+ with GPU_INFERENCE_LOCK, torch.inference_mode():
164
+ waveform_tensor = processor.decode(audio_codes)
165
+ waveform = waveform_tensor.detach().cpu().float().numpy().squeeze()
166
+ # Normalize to [-1, 1] range to prevent clipping/distortion
167
+ if waveform.ndim == 0:
168
+ waveform = None
169
+ elif len(waveform) > 0:
170
+ peak = np.abs(waveform).max()
171
+ if peak > 1.0:
172
+ waveform = waveform / peak * 0.95
173
+ elif peak < 0.01:
174
+ waveform = None # Too quiet, likely garbage
175
+ except Exception as e:
176
+ logger.warning("Audio decode failed: %s", e)
177
+ waveform = None
178
 
179
  return answer_text, waveform, SAMPLE_RATE
qa_flow.py CHANGED
@@ -68,8 +68,11 @@ def release_audio_submission(audio_path: str | Path | None) -> None:
68
 
69
  def _default_audio_writer(path: Path, waveform, sample_rate: int) -> None:
70
  import soundfile as sf
 
71
 
72
- sf.write(str(path), waveform, sample_rate)
 
 
73
 
74
 
75
  def _prune_generated_answers(output_dir: Path) -> None:
 
68
 
69
  def _default_audio_writer(path: Path, waveform, sample_rate: int) -> None:
70
  import soundfile as sf
71
+ import numpy as np
72
 
73
+ # Ensure float32 for proper WAV encoding
74
+ wav = np.asarray(waveform, dtype=np.float32)
75
+ sf.write(str(path), wav, sample_rate, subtype='FLOAT')
76
 
77
 
78
  def _prune_generated_answers(output_dir: Path) -> None: