Spaces:
Paused
Paused
Commit ·
215fff0
1
Parent(s): d70ebd4
feat: implement real voice cloning with Qwen3-TTS (sprint step 3)
Browse files- Add voice_clone.py: Qwen3-TTS model wrapper with server-side profile
cache, create_voice_profile() and synthesize_cloned() API
- Refactor tts.py: backend-neutral interface delegates to Qwen3-TTS
when voice_profile_id provided, Supertonic fallback otherwise
- Wire app.py: real cloning in animate_cloning_pipeline, voice profile
flows via gr.State to narration (stream_tts) and Q&A
- Fix XSS: html.escape in render_story_text and clone success card
- Fix misleading fallback: hide audio widget when TTS fails instead of
playing fake chime audio
- Add qwen-tts to requirements.txt
- Update copilot-instructions.md with new architecture
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
- .github/copilot-instructions.md +9 -6
- app.py +45 -31
- requirements.txt +4 -3
- sprint.md +1 -1
- test_modules/test_qwen3_tts_clone.py +107 -0
- tts.py +60 -16
- voice_clone.py +101 -0
.github/copilot-instructions.md
CHANGED
|
@@ -12,13 +12,16 @@ Requires a GPU (T4 or A10G) for inference. No Docker, no database, no external A
|
|
| 12 |
|
| 13 |
## Architecture
|
| 14 |
|
| 15 |
-
Single-process Gradio app (`app.py`, ~1200 lines) that orchestrates
|
| 16 |
|
| 17 |
-
- **
|
| 18 |
-
- **
|
| 19 |
-
- **
|
|
|
|
| 20 |
|
| 21 |
-
`tts.py`
|
|
|
|
|
|
|
| 22 |
|
| 23 |
Stories are plain `.txt` files in `stories/` — title on line 1, blank line, then prose. No metadata DB.
|
| 24 |
|
|
@@ -30,7 +33,7 @@ Stories are plain `.txt` files in `stories/` — title on line 1, blank line, th
|
|
| 30 |
- **In-memory session cache only** — no database, no persistent storage of user data.
|
| 31 |
- **Interruptible chunked streaming** — paragraphs are synthesized and played one at a time. Cached chunks enable instant replay/resume.
|
| 32 |
- **Pre-generated Q&A** — anticipated questions generated in background during narration for sub-1s cache hits.
|
| 33 |
-
- **VRAM budget awareness** — total ~
|
| 34 |
|
| 35 |
## Story Pipeline
|
| 36 |
|
|
|
|
| 12 |
|
| 13 |
## Architecture
|
| 14 |
|
| 15 |
+
Single-process Gradio app (`app.py`, ~1200 lines) that orchestrates four ML models:
|
| 16 |
|
| 17 |
+
- **Qwen3-TTS-1.7B** (`voice_clone.py`) — zero-shot voice cloning from reference audio + cloned voice synthesis
|
| 18 |
+
- **Supertonic TTS** — fast stock-voice fallback when no clone is available
|
| 19 |
+
- **Qwen2.5-3B-Instruct** (`inference.py`) — story Q&A, 4-bit quantized on T4
|
| 20 |
+
- **Whisper-small** (`inference.py`) — child speech-to-text (loaded on demand)
|
| 21 |
|
| 22 |
+
`tts.py` is the unified TTS interface. It delegates to Qwen3-TTS when a `voice_profile_id` is provided, otherwise falls back to Supertonic. Both backends use background-threaded streaming with a queue (maxsize=2 buffer).
|
| 23 |
+
|
| 24 |
+
`voice_clone.py` manages the Qwen3-TTS model and a server-side in-memory cache of voice profiles keyed by UUID. Profiles are created via `create_voice_profile(ref_audio_path)` and reused for all subsequent synthesis calls.
|
| 25 |
|
| 26 |
Stories are plain `.txt` files in `stories/` — title on line 1, blank line, then prose. No metadata DB.
|
| 27 |
|
|
|
|
| 33 |
- **In-memory session cache only** — no database, no persistent storage of user data.
|
| 34 |
- **Interruptible chunked streaming** — paragraphs are synthesized and played one at a time. Cached chunks enable instant replay/resume.
|
| 35 |
- **Pre-generated Q&A** — anticipated questions generated in background during narration for sub-1s cache hits.
|
| 36 |
+
- **VRAM budget awareness** — total ~8-9 GB on T4 (16 GB). All models lazy-loaded on demand. Use 4-bit quantization for the LLM. Qwen3-TTS (~1.7B) loads only when cloning starts.
|
| 37 |
|
| 38 |
## Story Pipeline
|
| 39 |
|
app.py
CHANGED
|
@@ -309,14 +309,15 @@ def load_paragraphs(story_path: str) -> list:
|
|
| 309 |
def render_story_text(paragraphs: list, current_idx: int) -> str:
|
| 310 |
if not paragraphs:
|
| 311 |
return ""
|
| 312 |
-
|
| 313 |
for i, para in enumerate(paragraphs):
|
|
|
|
| 314 |
if i == current_idx:
|
| 315 |
-
|
| 316 |
else:
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
return
|
| 320 |
|
| 321 |
def render_cloned_voices_html(voices_list):
|
| 322 |
html = """<div class="book-shelf-grid">"""
|
|
@@ -357,6 +358,7 @@ with gr.Blocks(css=css_code, title="VoiceBook Gradio Hub") as demo:
|
|
| 357 |
voices_state = gr.State(mock_voices)
|
| 358 |
paragraphs_state = gr.State([])
|
| 359 |
tts_chunks_state = gr.State([])
|
|
|
|
| 360 |
|
| 361 |
gr.HTML("""
|
| 362 |
<div style="display: flex; align-items: center; justify-content: space-between; padding: 16px 0; border-bottom: 1px solid #ebdccb; margin-bottom: 24px; flex-wrap: wrap; gap: 16px;">
|
|
@@ -681,34 +683,45 @@ with gr.Blocks(css=css_code, title="VoiceBook Gradio Hub") as demo:
|
|
| 681 |
if not v_name.strip():
|
| 682 |
return (
|
| 683 |
"""<div style="padding: 12px; background: #fef2f2; border: 1px solid #fecaca; color: #991b1b; border-radius: 12px; font-size: 12px; font-weight: 600;">
|
| 684 |
-
|
| 685 |
</div>""",
|
| 686 |
gr.HTML(visible=False),
|
| 687 |
gr.HTML(visible=False),
|
| 688 |
gr.Audio(visible=False),
|
| 689 |
-
gr.Button(visible=False)
|
|
|
|
| 690 |
)
|
| 691 |
if recorder_data is None:
|
| 692 |
return (
|
| 693 |
"""<div style="padding: 12px; background: #fef2f2; border: 1px solid #fecaca; color: #991b1b; border-radius: 12px; font-size: 12px; font-weight: 600;">
|
| 694 |
-
|
| 695 |
</div>""",
|
| 696 |
gr.HTML(visible=False),
|
| 697 |
gr.HTML(visible=False),
|
| 698 |
gr.Audio(visible=False),
|
| 699 |
-
gr.Button(visible=False)
|
|
|
|
| 700 |
)
|
| 701 |
|
| 702 |
-
|
| 703 |
-
|
| 704 |
-
|
| 705 |
-
|
| 706 |
-
|
| 707 |
-
|
| 708 |
-
progress(0.
|
| 709 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 710 |
|
| 711 |
-
|
| 712 |
avatar = "👩" if v_gender == "Female" else "👨"
|
| 713 |
|
| 714 |
cloned_card_html = f"""
|
|
@@ -718,12 +731,12 @@ with gr.Blocks(css=css_code, title="VoiceBook Gradio Hub") as demo:
|
|
| 718 |
{avatar}
|
| 719 |
</div>
|
| 720 |
<div>
|
| 721 |
-
<h4 class="serif-header" style="font-size: 16px; margin: 0;">{
|
| 722 |
-
<p style="font-size: 11px; color:#6f6257; margin-top:2px;">Synthesized today
|
| 723 |
</div>
|
| 724 |
</div>
|
| 725 |
<div style="background:#FAF7F2; border-left: 3px solid #f5841f; padding: 10px; font-family: Georgia, serif; font-style: italic; font-size: 12px; color: #1c1c19;">
|
| 726 |
-
|
| 727 |
</div>
|
| 728 |
</div>
|
| 729 |
"""
|
|
@@ -732,14 +745,15 @@ with gr.Blocks(css=css_code, title="VoiceBook Gradio Hub") as demo:
|
|
| 732 |
gr.HTML(visible=False),
|
| 733 |
gr.HTML(visible=False),
|
| 734 |
gr.HTML(value=cloned_card_html, visible=True),
|
| 735 |
-
gr.Audio(visible=
|
| 736 |
-
gr.Button(visible=True)
|
|
|
|
| 737 |
)
|
| 738 |
|
| 739 |
extract_btn.click(
|
| 740 |
animate_cloning_pipeline,
|
| 741 |
inputs=[new_voice_name, new_voice_gender, mic_recorder],
|
| 742 |
-
outputs=[cloning_progress_msg, loading_spinner, voice_cloning_success_panel, voice_sample_preview_widget, add_to_library_btn]
|
| 743 |
)
|
| 744 |
|
| 745 |
# 4. Add cloned voice to inventory
|
|
@@ -773,7 +787,7 @@ with gr.Blocks(css=css_code, title="VoiceBook Gradio Hub") as demo:
|
|
| 773 |
)
|
| 774 |
|
| 775 |
# 5. Play button — streams TTS audio chunk by chunk
|
| 776 |
-
def stream_tts(tts_chunks, paras):
|
| 777 |
_status_playing = """
|
| 778 |
<div style="margin-top: 12px; display: flex; align-items: center; gap: 10px; justify-content: center;">
|
| 779 |
<span style="width: 8px; height: 8px; border-radius: 50%; background: #4ade80; display: inline-block;"></span>
|
|
@@ -795,7 +809,7 @@ with gr.Blocks(css=css_code, title="VoiceBook Gradio Hub") as demo:
|
|
| 795 |
# Immediately show PLAYING state
|
| 796 |
yield None, _status_playing, f"<div style='text-align:center;font-size:10px;color:#4ade80;font-family:monospace;margin-top:8px;'>Generating chunk 1 / {n}…</div>", gr.Button(visible=False), gr.Button(visible=True), gr.Button(visible=False)
|
| 797 |
|
| 798 |
-
for sample_rate, wav, i, total, err in generate_audio_stream(tts_chunks):
|
| 799 |
if err:
|
| 800 |
yield None, f"<div style='color:#ef4444;font-size:11px;'>Error on chunk {i+1}: {err}</div>", "", gr.Button(visible=True), gr.Button(visible=False), gr.Button(visible=False)
|
| 801 |
return
|
|
@@ -808,7 +822,7 @@ with gr.Blocks(css=css_code, title="VoiceBook Gradio Hub") as demo:
|
|
| 808 |
|
| 809 |
play_btn.click(
|
| 810 |
stream_tts,
|
| 811 |
-
inputs=[tts_chunks_state, paragraphs_state],
|
| 812 |
outputs=[player_audio_control, player_status_bar, chunk_status, play_btn, pause_btn, ask_btn]
|
| 813 |
)
|
| 814 |
|
|
@@ -871,7 +885,7 @@ with gr.Blocks(css=css_code, title="VoiceBook Gradio Hub") as demo:
|
|
| 871 |
)
|
| 872 |
|
| 873 |
# 9. Submit question
|
| 874 |
-
def handle_question_submit(question_txt, question_audio_path, paragraphs, slider_val):
|
| 875 |
if not question_txt.strip() and question_audio_path is None:
|
| 876 |
answer_html = """
|
| 877 |
<div style="padding: 10px; background: #fef2f2; border: 1px solid #fecaca; border-radius: 10px; font-size: 12px; color: #991b1b;">
|
|
@@ -913,7 +927,7 @@ with gr.Blocks(css=css_code, title="VoiceBook Gradio Hub") as demo:
|
|
| 913 |
chunks = _split(answer_text)
|
| 914 |
audio_segments = []
|
| 915 |
sample_rate = 16000
|
| 916 |
-
for sr, wav, idx, total, err in _gen_stream(chunks):
|
| 917 |
if wav is not None:
|
| 918 |
audio_segments.append(wav)
|
| 919 |
sample_rate = sr
|
|
@@ -940,11 +954,11 @@ with gr.Blocks(css=css_code, title="VoiceBook Gradio Hub") as demo:
|
|
| 940 |
"""
|
| 941 |
if answer_audio_path:
|
| 942 |
return answer_html, gr.Audio(value=answer_audio_path, visible=True)
|
| 943 |
-
return answer_html, gr.Audio(
|
| 944 |
|
| 945 |
submit_question_btn.click(
|
| 946 |
handle_question_submit,
|
| 947 |
-
inputs=[question_text, question_audio, paragraphs_state, timeline_slider],
|
| 948 |
outputs=[answer_display, answer_audio]
|
| 949 |
)
|
| 950 |
|
|
|
|
| 309 |
def render_story_text(paragraphs: list, current_idx: int) -> str:
|
| 310 |
if not paragraphs:
|
| 311 |
return ""
|
| 312 |
+
result = """<div style="max-height: 420px; overflow-y: auto; padding: 4px 2px; margin-top: 12px;">"""
|
| 313 |
for i, para in enumerate(paragraphs):
|
| 314 |
+
safe_para = html.escape(para)
|
| 315 |
if i == current_idx:
|
| 316 |
+
result += f"""<p style="font-family: 'Playfair Display', Georgia, serif; font-size: 13px; line-height: 1.8; color: #FAF7F2; background: rgba(245,132,31,0.18); border-left: 3px solid #f5841f; padding: 8px 12px; border-radius: 6px; margin: 6px 0;">{safe_para}</p>"""
|
| 317 |
else:
|
| 318 |
+
result += f"""<p style="font-family: 'Playfair Display', Georgia, serif; font-size: 13px; line-height: 1.8; color: #94a3b8; padding: 4px 12px; margin: 4px 0;">{safe_para}</p>"""
|
| 319 |
+
result += "</div>"
|
| 320 |
+
return result
|
| 321 |
|
| 322 |
def render_cloned_voices_html(voices_list):
|
| 323 |
html = """<div class="book-shelf-grid">"""
|
|
|
|
| 358 |
voices_state = gr.State(mock_voices)
|
| 359 |
paragraphs_state = gr.State([])
|
| 360 |
tts_chunks_state = gr.State([])
|
| 361 |
+
voice_profile_state = gr.State(None)
|
| 362 |
|
| 363 |
gr.HTML("""
|
| 364 |
<div style="display: flex; align-items: center; justify-content: space-between; padding: 16px 0; border-bottom: 1px solid #ebdccb; margin-bottom: 24px; flex-wrap: wrap; gap: 16px;">
|
|
|
|
| 683 |
if not v_name.strip():
|
| 684 |
return (
|
| 685 |
"""<div style="padding: 12px; background: #fef2f2; border: 1px solid #fecaca; color: #991b1b; border-radius: 12px; font-size: 12px; font-weight: 600;">
|
| 686 |
+
Please provide a comforting nickname for your cloning candidate.
|
| 687 |
</div>""",
|
| 688 |
gr.HTML(visible=False),
|
| 689 |
gr.HTML(visible=False),
|
| 690 |
gr.Audio(visible=False),
|
| 691 |
+
gr.Button(visible=False),
|
| 692 |
+
None,
|
| 693 |
)
|
| 694 |
if recorder_data is None:
|
| 695 |
return (
|
| 696 |
"""<div style="padding: 12px; background: #fef2f2; border: 1px solid #fecaca; color: #991b1b; border-radius: 12px; font-size: 12px; font-weight: 600;">
|
| 697 |
+
Microphone recording sample missing. Speak some script lines!
|
| 698 |
</div>""",
|
| 699 |
gr.HTML(visible=False),
|
| 700 |
gr.HTML(visible=False),
|
| 701 |
gr.Audio(visible=False),
|
| 702 |
+
gr.Button(visible=False),
|
| 703 |
+
None,
|
| 704 |
)
|
| 705 |
|
| 706 |
+
from voice_clone import create_voice_profile, synthesize_cloned_preview
|
| 707 |
+
import soundfile as sf
|
| 708 |
+
|
| 709 |
+
progress(0.1, desc="Extracting speaker embedding from recording...")
|
| 710 |
+
profile_id = create_voice_profile(recorder_data)
|
| 711 |
+
|
| 712 |
+
progress(0.7, desc="Generating voice preview...")
|
| 713 |
+
try:
|
| 714 |
+
preview_wav, preview_sr = synthesize_cloned_preview(profile_id)
|
| 715 |
+
preview_path = f"sample_sounds/clone_preview_{profile_id}.wav"
|
| 716 |
+
sf.write(preview_path, preview_wav, preview_sr)
|
| 717 |
+
except Exception as e:
|
| 718 |
+
import logging
|
| 719 |
+
logging.getLogger(__name__).exception("Preview synthesis failed")
|
| 720 |
+
preview_path = None
|
| 721 |
+
|
| 722 |
+
progress(1.0, desc="Voice cloned successfully!")
|
| 723 |
|
| 724 |
+
safe_name = html.escape(v_name)
|
| 725 |
avatar = "👩" if v_gender == "Female" else "👨"
|
| 726 |
|
| 727 |
cloned_card_html = f"""
|
|
|
|
| 731 |
{avatar}
|
| 732 |
</div>
|
| 733 |
<div>
|
| 734 |
+
<h4 class="serif-header" style="font-size: 16px; margin: 0;">{safe_name} <span style="font-size: 9px; background: #f0fdf4; color: #16a34a; padding: 2px 6px; border-radius: 4px; font-weight:700;">Cloned successfully</span></h4>
|
| 735 |
+
<p style="font-size: 11px; color:#6f6257; margin-top:2px;">Synthesized today using Qwen3-TTS voice cloning</p>
|
| 736 |
</div>
|
| 737 |
</div>
|
| 738 |
<div style="background:#FAF7F2; border-left: 3px solid #f5841f; padding: 10px; font-family: Georgia, serif; font-style: italic; font-size: 12px; color: #1c1c19;">
|
| 739 |
+
Voice profile ready. Select a story and tap Play to hear narration in this voice.
|
| 740 |
</div>
|
| 741 |
</div>
|
| 742 |
"""
|
|
|
|
| 745 |
gr.HTML(visible=False),
|
| 746 |
gr.HTML(visible=False),
|
| 747 |
gr.HTML(value=cloned_card_html, visible=True),
|
| 748 |
+
gr.Audio(value=preview_path, visible=preview_path is not None),
|
| 749 |
+
gr.Button(visible=True),
|
| 750 |
+
profile_id,
|
| 751 |
)
|
| 752 |
|
| 753 |
extract_btn.click(
|
| 754 |
animate_cloning_pipeline,
|
| 755 |
inputs=[new_voice_name, new_voice_gender, mic_recorder],
|
| 756 |
+
outputs=[cloning_progress_msg, loading_spinner, voice_cloning_success_panel, voice_sample_preview_widget, add_to_library_btn, voice_profile_state]
|
| 757 |
)
|
| 758 |
|
| 759 |
# 4. Add cloned voice to inventory
|
|
|
|
| 787 |
)
|
| 788 |
|
| 789 |
# 5. Play button — streams TTS audio chunk by chunk
|
| 790 |
+
def stream_tts(tts_chunks, paras, profile_id):
|
| 791 |
_status_playing = """
|
| 792 |
<div style="margin-top: 12px; display: flex; align-items: center; gap: 10px; justify-content: center;">
|
| 793 |
<span style="width: 8px; height: 8px; border-radius: 50%; background: #4ade80; display: inline-block;"></span>
|
|
|
|
| 809 |
# Immediately show PLAYING state
|
| 810 |
yield None, _status_playing, f"<div style='text-align:center;font-size:10px;color:#4ade80;font-family:monospace;margin-top:8px;'>Generating chunk 1 / {n}…</div>", gr.Button(visible=False), gr.Button(visible=True), gr.Button(visible=False)
|
| 811 |
|
| 812 |
+
for sample_rate, wav, i, total, err in generate_audio_stream(tts_chunks, voice_profile_id=profile_id):
|
| 813 |
if err:
|
| 814 |
yield None, f"<div style='color:#ef4444;font-size:11px;'>Error on chunk {i+1}: {err}</div>", "", gr.Button(visible=True), gr.Button(visible=False), gr.Button(visible=False)
|
| 815 |
return
|
|
|
|
| 822 |
|
| 823 |
play_btn.click(
|
| 824 |
stream_tts,
|
| 825 |
+
inputs=[tts_chunks_state, paragraphs_state, voice_profile_state],
|
| 826 |
outputs=[player_audio_control, player_status_bar, chunk_status, play_btn, pause_btn, ask_btn]
|
| 827 |
)
|
| 828 |
|
|
|
|
| 885 |
)
|
| 886 |
|
| 887 |
# 9. Submit question
|
| 888 |
+
def handle_question_submit(question_txt, question_audio_path, paragraphs, slider_val, profile_id):
|
| 889 |
if not question_txt.strip() and question_audio_path is None:
|
| 890 |
answer_html = """
|
| 891 |
<div style="padding: 10px; background: #fef2f2; border: 1px solid #fecaca; border-radius: 10px; font-size: 12px; color: #991b1b;">
|
|
|
|
| 927 |
chunks = _split(answer_text)
|
| 928 |
audio_segments = []
|
| 929 |
sample_rate = 16000
|
| 930 |
+
for sr, wav, idx, total, err in _gen_stream(chunks, voice_profile_id=profile_id):
|
| 931 |
if wav is not None:
|
| 932 |
audio_segments.append(wav)
|
| 933 |
sample_rate = sr
|
|
|
|
| 954 |
"""
|
| 955 |
if answer_audio_path:
|
| 956 |
return answer_html, gr.Audio(value=answer_audio_path, visible=True)
|
| 957 |
+
return answer_html, gr.Audio(visible=False)
|
| 958 |
|
| 959 |
submit_question_btn.click(
|
| 960 |
handle_question_submit,
|
| 961 |
+
inputs=[question_text, question_audio, paragraphs_state, timeline_slider, voice_profile_state],
|
| 962 |
outputs=[answer_display, answer_audio]
|
| 963 |
)
|
| 964 |
|
requirements.txt
CHANGED
|
@@ -6,9 +6,10 @@ bitsandbytes
|
|
| 6 |
soundfile
|
| 7 |
numpy
|
| 8 |
|
| 9 |
-
|
| 10 |
-
# Supertonic TTS
|
| 11 |
supertonic>=1.3.1
|
| 12 |
-
soundfile>=0.12.0
|
| 13 |
onnxruntime>=1.18.0
|
| 14 |
huggingface-hub>=0.23.0
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
soundfile
|
| 7 |
numpy
|
| 8 |
|
| 9 |
+
# Supertonic TTS (fallback voice)
|
|
|
|
| 10 |
supertonic>=1.3.1
|
|
|
|
| 11 |
onnxruntime>=1.18.0
|
| 12 |
huggingface-hub>=0.23.0
|
| 13 |
+
|
| 14 |
+
# Qwen3-TTS (voice cloning)
|
| 15 |
+
qwen-tts>=0.1.1
|
sprint.md
CHANGED
|
@@ -13,7 +13,7 @@ Ship a public Hugging Face Space: parent clones voice → story streams in that
|
|
| 13 |
|---|---|---|---|
|
| 14 |
| 1 | Set up repo: `app.py`, `requirements.txt`, `stories/` | 30m | ☑ |
|
| 15 |
| 2 | Load QWEN-TTS-0.6B locally, test basic TTS (text → audio) | 1h | ☑ | *(switched to Supertonic TTS; wired in `tts.py`, tested in `test_modules/`)* |
|
| 16 |
-
| 3 | Implement voice cloning and cache voice representation after recording | 1.5h |
|
| 17 |
| 4 | Add 10 short stories as `.txt` files (public domain, from Project Gutenberg) | 30m | ☑ |
|
| 18 |
| 5 | Wire up: pick story → stream first narration chunk, track chunk index, cache story audio | 1h | ☑ | *(`handle_book_select` loads paragraphs + splits chunks; `stream_tts` generator streams audio with play/pause/chunk-index tracking)* |
|
| 19 |
|
|
|
|
| 13 |
|---|---|---|---|
|
| 14 |
| 1 | Set up repo: `app.py`, `requirements.txt`, `stories/` | 30m | ☑ |
|
| 15 |
| 2 | Load QWEN-TTS-0.6B locally, test basic TTS (text → audio) | 1h | ☑ | *(switched to Supertonic TTS; wired in `tts.py`, tested in `test_modules/`)* |
|
| 16 |
+
| 3 | Implement voice cloning and cache voice representation after recording | 1.5h | ☑ | *(Qwen3-TTS voice cloning in `voice_clone.py`; `create_voice_profile()` extracts speaker embedding, cached server-side; preview synthesis on clone; profile_id flows via `gr.State` to narration + Q&A)* |
|
| 17 |
| 4 | Add 10 short stories as `.txt` files (public domain, from Project Gutenberg) | 30m | ☑ |
|
| 18 |
| 5 | Wire up: pick story → stream first narration chunk, track chunk index, cache story audio | 1h | ☑ | *(`handle_book_select` loads paragraphs + splits chunks; `stream_tts` generator streams audio with play/pause/chunk-index tracking)* |
|
| 19 |
|
test_modules/test_qwen3_tts_clone.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Test Qwen3-TTS voice cloning: load model, extract voice from reference audio, synthesize.
|
| 3 |
+
Usage: python test_modules/test_qwen3_tts_clone.py
|
| 4 |
+
"""
|
| 5 |
+
import sys
|
| 6 |
+
import time
|
| 7 |
+
import numpy as np
|
| 8 |
+
import soundfile as sf
|
| 9 |
+
|
| 10 |
+
print("=" * 60)
|
| 11 |
+
print("Testing Qwen3-TTS Voice Cloning")
|
| 12 |
+
print("=" * 60)
|
| 13 |
+
|
| 14 |
+
# Generate a synthetic reference audio (sine wave simulating speech duration)
|
| 15 |
+
# In production this would be the parent's recorded voice
|
| 16 |
+
print("Creating synthetic reference audio for testing...")
|
| 17 |
+
sr = 16000
|
| 18 |
+
duration = 5 # 5 seconds
|
| 19 |
+
t = np.linspace(0, duration, sr * duration, dtype=np.float32)
|
| 20 |
+
# Simple multi-frequency signal (not real speech, but tests the pipeline)
|
| 21 |
+
ref_audio = 0.3 * np.sin(2 * np.pi * 200 * t) + 0.2 * np.sin(2 * np.pi * 400 * t)
|
| 22 |
+
ref_audio_path = "sample_sounds/test_ref_audio.wav"
|
| 23 |
+
sf.write(ref_audio_path, ref_audio, sr)
|
| 24 |
+
print(f"[OK] Reference audio created: {ref_audio_path} ({duration}s)")
|
| 25 |
+
|
| 26 |
+
print()
|
| 27 |
+
print("Loading Qwen3-TTS model (this downloads ~1.7GB on first run)...")
|
| 28 |
+
start = time.time()
|
| 29 |
+
|
| 30 |
+
try:
|
| 31 |
+
import torch
|
| 32 |
+
from qwen_tts import Qwen3TTSModel
|
| 33 |
+
|
| 34 |
+
model = Qwen3TTSModel.from_pretrained(
|
| 35 |
+
"Qwen/Qwen3-TTS-12Hz-1.7B-Base",
|
| 36 |
+
device_map="cuda" if torch.cuda.is_available() else "cpu",
|
| 37 |
+
torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
|
| 38 |
+
)
|
| 39 |
+
elapsed = time.time() - start
|
| 40 |
+
print(f"[OK] Qwen3-TTS loaded in {elapsed:.1f}s")
|
| 41 |
+
except Exception as e:
|
| 42 |
+
print(f"[FAIL] Model loading failed: {e}")
|
| 43 |
+
import traceback
|
| 44 |
+
traceback.print_exc()
|
| 45 |
+
sys.exit(1)
|
| 46 |
+
|
| 47 |
+
# Test 1: Voice cloning with x_vector_only_mode (speaker embedding only, no ref_text needed)
|
| 48 |
+
print()
|
| 49 |
+
print("Test 1: Voice clone with x_vector_only_mode=True...")
|
| 50 |
+
try:
|
| 51 |
+
start = time.time()
|
| 52 |
+
audio_list, sample_rate = model.generate_voice_clone(
|
| 53 |
+
text="Hello! I am reading a bedtime story for you tonight.",
|
| 54 |
+
language="en",
|
| 55 |
+
ref_audio=ref_audio_path,
|
| 56 |
+
x_vector_only_mode=True,
|
| 57 |
+
)
|
| 58 |
+
elapsed = time.time() - start
|
| 59 |
+
total_samples = sum(len(a) for a in audio_list)
|
| 60 |
+
duration_out = total_samples / sample_rate
|
| 61 |
+
print(f"[OK] Voice clone (x_vector) in {elapsed:.1f}s, output: {duration_out:.1f}s at {sample_rate}Hz")
|
| 62 |
+
|
| 63 |
+
# Save output
|
| 64 |
+
output = np.concatenate(audio_list)
|
| 65 |
+
sf.write("sample_sounds/test_clone_xvector.wav", output, sample_rate)
|
| 66 |
+
print(f"[OK] Saved to sample_sounds/test_clone_xvector.wav")
|
| 67 |
+
except Exception as e:
|
| 68 |
+
print(f"[FAIL] x_vector clone failed: {e}")
|
| 69 |
+
import traceback
|
| 70 |
+
traceback.print_exc()
|
| 71 |
+
|
| 72 |
+
# Test 2: Create and reuse voice clone prompt (for caching)
|
| 73 |
+
print()
|
| 74 |
+
print("Test 2: Create reusable voice clone prompt...")
|
| 75 |
+
try:
|
| 76 |
+
start = time.time()
|
| 77 |
+
prompt_items = model.create_voice_clone_prompt(
|
| 78 |
+
ref_audio=ref_audio_path,
|
| 79 |
+
x_vector_only_mode=True,
|
| 80 |
+
)
|
| 81 |
+
elapsed = time.time() - start
|
| 82 |
+
print(f"[OK] Voice clone prompt created in {elapsed:.1f}s")
|
| 83 |
+
|
| 84 |
+
# Reuse cached prompt for synthesis
|
| 85 |
+
start = time.time()
|
| 86 |
+
audio_list2, sr2 = model.generate_voice_clone(
|
| 87 |
+
text="Once upon a time, there was a little rabbit named Peter.",
|
| 88 |
+
language="en",
|
| 89 |
+
voice_clone_prompt=prompt_items,
|
| 90 |
+
)
|
| 91 |
+
elapsed = time.time() - start
|
| 92 |
+
total_samples2 = sum(len(a) for a in audio_list2)
|
| 93 |
+
duration_out2 = total_samples2 / sr2
|
| 94 |
+
print(f"[OK] Synthesis from cached prompt in {elapsed:.1f}s, output: {duration_out2:.1f}s")
|
| 95 |
+
|
| 96 |
+
output2 = np.concatenate(audio_list2)
|
| 97 |
+
sf.write("sample_sounds/test_clone_cached.wav", output2, sr2)
|
| 98 |
+
print(f"[OK] Saved to sample_sounds/test_clone_cached.wav")
|
| 99 |
+
except Exception as e:
|
| 100 |
+
print(f"[FAIL] Cached prompt failed: {e}")
|
| 101 |
+
import traceback
|
| 102 |
+
traceback.print_exc()
|
| 103 |
+
|
| 104 |
+
print()
|
| 105 |
+
print("=" * 60)
|
| 106 |
+
print("[OK] Qwen3-TTS voice cloning tests complete")
|
| 107 |
+
print("=" * 60)
|
tts.py
CHANGED
|
@@ -1,12 +1,22 @@
|
|
| 1 |
"""
|
| 2 |
-
TTS module —
|
| 3 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
"""
|
| 5 |
import logging
|
| 6 |
import queue
|
| 7 |
import re
|
| 8 |
import threading
|
| 9 |
|
|
|
|
|
|
|
| 10 |
logger = logging.getLogger(__name__)
|
| 11 |
|
| 12 |
_SENTENCE_RE = re.compile(r'(?<=[.!?;])\s+')
|
|
@@ -15,11 +25,11 @@ _SENTINEL = object()
|
|
| 15 |
_tts_instance = None
|
| 16 |
|
| 17 |
|
| 18 |
-
def
|
| 19 |
global _tts_instance
|
| 20 |
if _tts_instance is None:
|
| 21 |
from supertonic import TTS
|
| 22 |
-
logger.info("Loading Supertonic TTS model
|
| 23 |
_tts_instance = TTS(auto_download=True)
|
| 24 |
return _tts_instance
|
| 25 |
|
|
@@ -30,33 +40,67 @@ def split_into_chunks(text: str) -> list[str]:
|
|
| 30 |
return [p.strip() for p in parts if p.strip()]
|
| 31 |
|
| 32 |
|
| 33 |
-
def generate_audio_stream(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
"""
|
| 35 |
Generator: synthesizes chunks in a background thread (maxsize=2 buffer).
|
| 36 |
Yields (sample_rate, wav_array, chunk_idx, total_chunks, error_msg).
|
| 37 |
-
|
|
|
|
|
|
|
| 38 |
"""
|
| 39 |
-
tts = get_tts()
|
| 40 |
-
style = tts.get_voice_style(voice_name)
|
| 41 |
n = len(chunks)
|
| 42 |
chunk_q: queue.Queue = queue.Queue(maxsize=2)
|
| 43 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
def _worker():
|
| 45 |
for i, stmt in enumerate(chunks):
|
| 46 |
try:
|
| 47 |
wav, _ = tts.synthesize(stmt, voice_style=style)
|
| 48 |
chunk_q.put((i, wav.squeeze(), None))
|
| 49 |
except Exception as exc:
|
| 50 |
-
logger.exception("
|
| 51 |
chunk_q.put((i, None, str(exc)))
|
| 52 |
return
|
| 53 |
chunk_q.put(_SENTINEL)
|
| 54 |
|
| 55 |
threading.Thread(target=_worker, daemon=True).start()
|
| 56 |
-
|
| 57 |
-
while True:
|
| 58 |
-
item = chunk_q.get()
|
| 59 |
-
if item is _SENTINEL:
|
| 60 |
-
break
|
| 61 |
-
i, wav, err = item
|
| 62 |
-
yield tts.sample_rate, wav, i, n, err
|
|
|
|
| 1 |
"""
|
| 2 |
+
TTS module — unified interface for text-to-speech synthesis.
|
| 3 |
+
|
| 4 |
+
Supports two backends:
|
| 5 |
+
- Qwen3-TTS: voice-cloned synthesis using a cached voice profile
|
| 6 |
+
- Supertonic: fast stock voice fallback (no cloning)
|
| 7 |
+
|
| 8 |
+
Usage:
|
| 9 |
+
chunks = split_into_chunks(text)
|
| 10 |
+
for sr, wav, i, n, err in generate_audio_stream(chunks, voice_profile_id="abc123"):
|
| 11 |
+
...
|
| 12 |
"""
|
| 13 |
import logging
|
| 14 |
import queue
|
| 15 |
import re
|
| 16 |
import threading
|
| 17 |
|
| 18 |
+
import numpy as np
|
| 19 |
+
|
| 20 |
logger = logging.getLogger(__name__)
|
| 21 |
|
| 22 |
_SENTENCE_RE = re.compile(r'(?<=[.!?;])\s+')
|
|
|
|
| 25 |
_tts_instance = None
|
| 26 |
|
| 27 |
|
| 28 |
+
def _get_supertonic():
|
| 29 |
global _tts_instance
|
| 30 |
if _tts_instance is None:
|
| 31 |
from supertonic import TTS
|
| 32 |
+
logger.info("Loading Supertonic TTS model...")
|
| 33 |
_tts_instance = TTS(auto_download=True)
|
| 34 |
return _tts_instance
|
| 35 |
|
|
|
|
| 40 |
return [p.strip() for p in parts if p.strip()]
|
| 41 |
|
| 42 |
|
| 43 |
+
def generate_audio_stream(
|
| 44 |
+
chunks: list[str],
|
| 45 |
+
voice_profile_id: str | None = None,
|
| 46 |
+
voice_name: str = "F1",
|
| 47 |
+
):
|
| 48 |
"""
|
| 49 |
Generator: synthesizes chunks in a background thread (maxsize=2 buffer).
|
| 50 |
Yields (sample_rate, wav_array, chunk_idx, total_chunks, error_msg).
|
| 51 |
+
|
| 52 |
+
If voice_profile_id is provided, uses Qwen3-TTS with the cloned voice.
|
| 53 |
+
Otherwise falls back to Supertonic with the given voice_name.
|
| 54 |
"""
|
|
|
|
|
|
|
| 55 |
n = len(chunks)
|
| 56 |
chunk_q: queue.Queue = queue.Queue(maxsize=2)
|
| 57 |
|
| 58 |
+
if voice_profile_id:
|
| 59 |
+
_start_qwen_worker(chunks, voice_profile_id, chunk_q)
|
| 60 |
+
sample_rate = 24000 # Qwen3-TTS output rate
|
| 61 |
+
else:
|
| 62 |
+
sample_rate = _start_supertonic_worker(chunks, voice_name, chunk_q)
|
| 63 |
+
|
| 64 |
+
while True:
|
| 65 |
+
item = chunk_q.get()
|
| 66 |
+
if item is _SENTINEL:
|
| 67 |
+
break
|
| 68 |
+
i, wav, err = item
|
| 69 |
+
yield sample_rate, wav, i, n, err
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def _start_qwen_worker(chunks, profile_id, chunk_q):
|
| 73 |
+
"""Background thread: synthesize chunks with Qwen3-TTS voice clone."""
|
| 74 |
+
def _worker():
|
| 75 |
+
from voice_clone import synthesize_cloned
|
| 76 |
+
for i, stmt in enumerate(chunks):
|
| 77 |
+
try:
|
| 78 |
+
wav, _sr = synthesize_cloned(stmt, profile_id)
|
| 79 |
+
chunk_q.put((i, wav, None))
|
| 80 |
+
except Exception as exc:
|
| 81 |
+
logger.exception("Qwen3-TTS synthesis failed on chunk %d", i)
|
| 82 |
+
chunk_q.put((i, None, str(exc)))
|
| 83 |
+
return
|
| 84 |
+
chunk_q.put(_SENTINEL)
|
| 85 |
+
|
| 86 |
+
threading.Thread(target=_worker, daemon=True).start()
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def _start_supertonic_worker(chunks, voice_name, chunk_q):
|
| 90 |
+
"""Background thread: synthesize chunks with Supertonic (stock voice)."""
|
| 91 |
+
tts = _get_supertonic()
|
| 92 |
+
style = tts.get_voice_style(voice_name)
|
| 93 |
+
|
| 94 |
def _worker():
|
| 95 |
for i, stmt in enumerate(chunks):
|
| 96 |
try:
|
| 97 |
wav, _ = tts.synthesize(stmt, voice_style=style)
|
| 98 |
chunk_q.put((i, wav.squeeze(), None))
|
| 99 |
except Exception as exc:
|
| 100 |
+
logger.exception("Supertonic synthesis failed on chunk %d", i)
|
| 101 |
chunk_q.put((i, None, str(exc)))
|
| 102 |
return
|
| 103 |
chunk_q.put(_SENTINEL)
|
| 104 |
|
| 105 |
threading.Thread(target=_worker, daemon=True).start()
|
| 106 |
+
return tts.sample_rate
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
voice_clone.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Voice cloning module — wraps Qwen3-TTS for zero-shot voice cloning.
|
| 3 |
+
|
| 4 |
+
Usage:
|
| 5 |
+
profile_id = create_voice_profile(ref_audio_path)
|
| 6 |
+
wav, sr = synthesize_cloned(text, profile_id)
|
| 7 |
+
"""
|
| 8 |
+
import logging
|
| 9 |
+
import uuid
|
| 10 |
+
import threading
|
| 11 |
+
|
| 12 |
+
import numpy as np
|
| 13 |
+
import torch
|
| 14 |
+
|
| 15 |
+
logger = logging.getLogger(__name__)
|
| 16 |
+
|
| 17 |
+
# Server-side cache: { profile_id -> VoiceClonePromptItem list }
|
| 18 |
+
_PROFILE_CACHE: dict[str, list] = {}
|
| 19 |
+
_cache_lock = threading.Lock()
|
| 20 |
+
|
| 21 |
+
_qwen_tts_model = None
|
| 22 |
+
_model_lock = threading.Lock()
|
| 23 |
+
|
| 24 |
+
QWEN_TTS_MODEL_ID = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def get_qwen_tts():
|
| 28 |
+
"""Lazy-load Qwen3-TTS model. Thread-safe, cached globally."""
|
| 29 |
+
global _qwen_tts_model
|
| 30 |
+
if _qwen_tts_model is None:
|
| 31 |
+
with _model_lock:
|
| 32 |
+
if _qwen_tts_model is None:
|
| 33 |
+
from qwen_tts import Qwen3TTSModel
|
| 34 |
+
|
| 35 |
+
logger.info("Loading %s...", QWEN_TTS_MODEL_ID)
|
| 36 |
+
_qwen_tts_model = Qwen3TTSModel.from_pretrained(
|
| 37 |
+
QWEN_TTS_MODEL_ID,
|
| 38 |
+
device_map="cuda" if torch.cuda.is_available() else "cpu",
|
| 39 |
+
torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
|
| 40 |
+
)
|
| 41 |
+
logger.info("Qwen3-TTS loaded.")
|
| 42 |
+
return _qwen_tts_model
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def create_voice_profile(ref_audio_path: str) -> str:
|
| 46 |
+
"""
|
| 47 |
+
Extract speaker embedding from reference audio and cache it.
|
| 48 |
+
Returns a profile_id string for later synthesis.
|
| 49 |
+
"""
|
| 50 |
+
model = get_qwen_tts()
|
| 51 |
+
|
| 52 |
+
logger.info("Creating voice profile from %s...", ref_audio_path)
|
| 53 |
+
prompt_items = model.create_voice_clone_prompt(
|
| 54 |
+
ref_audio=ref_audio_path,
|
| 55 |
+
x_vector_only_mode=True,
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
profile_id = uuid.uuid4().hex[:12]
|
| 59 |
+
with _cache_lock:
|
| 60 |
+
_PROFILE_CACHE[profile_id] = prompt_items
|
| 61 |
+
|
| 62 |
+
logger.info("Voice profile %s created.", profile_id)
|
| 63 |
+
return profile_id
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def synthesize_cloned(text: str, profile_id: str) -> tuple[np.ndarray, int]:
|
| 67 |
+
"""
|
| 68 |
+
Synthesize text using a cached voice profile.
|
| 69 |
+
Returns (wav_array, sample_rate).
|
| 70 |
+
"""
|
| 71 |
+
with _cache_lock:
|
| 72 |
+
prompt_items = _PROFILE_CACHE.get(profile_id)
|
| 73 |
+
if prompt_items is None:
|
| 74 |
+
raise ValueError(f"Voice profile '{profile_id}' not found. Record voice first.")
|
| 75 |
+
|
| 76 |
+
model = get_qwen_tts()
|
| 77 |
+
|
| 78 |
+
audio_list, sample_rate = model.generate_voice_clone(
|
| 79 |
+
text=text,
|
| 80 |
+
language="english",
|
| 81 |
+
voice_clone_prompt=prompt_items,
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
wav = np.concatenate(audio_list) if audio_list else np.zeros(0, dtype=np.float32)
|
| 85 |
+
return wav, sample_rate
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def synthesize_cloned_preview(profile_id: str) -> tuple[np.ndarray, int]:
|
| 89 |
+
"""Short preview sentence to verify the clone sounds right."""
|
| 90 |
+
return synthesize_cloned(
|
| 91 |
+
"Hello! I'm ready to read a bedtime story for you tonight.",
|
| 92 |
+
profile_id,
|
| 93 |
+
)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def has_profile(profile_id: str | None) -> bool:
|
| 97 |
+
"""Check if a voice profile exists in cache."""
|
| 98 |
+
if not profile_id:
|
| 99 |
+
return False
|
| 100 |
+
with _cache_lock:
|
| 101 |
+
return profile_id in _PROFILE_CACHE
|