player: FluidSynth (GM soundfont) audio backend with piano fallback

Replace the single-piano MIDI.js soundfont with FluidSynth (js-synthesizer
WASM) playing a 128-instrument General MIDI gm.sf3, so each MIDI program
sounds with its own timbre instead of everything-as-piano. The legacy
grand-piano soundfont stays as an immediate fallback while the ~40MB GM
font downloads, then FluidSynth takes over transparently.

- web/fluid-audio.js: vanilla-JS port of live-editor's fluidAudio.ts,
window.LilyFluidAudio (init/loadPlugin/resume/noteOn/noteOff/
programChange/stopAllNotes), routing to FluidSynth's Sequencer with the
legacy MidiAudio piano as fallback.
- web/fluid/, web/vendor/js-synthesizer.min.js, web/soundfont/gm.sf3:
vendored FluidSynth runtime + GM soundfont (LFS).
- score-player.js: initAudio drives LilyFluidAudio; reports ready as soon
as any backend is audible; play() resume()s the AudioContext. Adds a
sound-library status badge (pulsing while loading, steady green when
ready).
- app.py: build_head injects js-synthesizer + fluid-audio.js and the
fluid/soundfont base-URL globals.

.gitattributes

@@ -39,3 +39,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 # stays a normal text file.
 web/vendor/*.js filter=lfs diff=lfs merge=lfs -text
 web/soundfont/*.js filter=lfs diff=lfs merge=lfs -text
+# FluidSynth (js-synthesizer) runtime + General-MIDI soundfont for the player.
+web/fluid/*.js filter=lfs diff=lfs merge=lfs -text
+web/soundfont/*.sf3 filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -14,3 +14,4 @@ __pycache__/
 .DS_Store
 
 *.local
+*.bak-*

app.py

@@ -339,14 +339,21 @@ def build_head ():
 		os.path.join(vendor, 'lilylet.bundle.js'),
 		os.path.join(vendor, 'verovio.bundle.js'),
 		os.path.join(vendor, 'musicWidgetsBrowser.umd.min.js'),
+		# FluidSynth audio backend: js-synthesizer UMD (window.JSSynth) + our adapter
+		# (window.LilyFluidAudio). Must precede score-player.js, which calls
+		# LilyFluidAudio.init() in initAudio(). The libfluidsynth runtime + gm.sf3 are
+		# loaded on demand by the adapter from web/fluid/ and web/soundfont/.
+		os.path.join(vendor, 'js-synthesizer.min.js'),
+		os.path.join(WEB_DIR, 'fluid-audio.js'),
 		os.path.join(WEB_DIR, 'score-player.js'),
 		# our own CodeMirror 6 editor (CM + grammar-derived lilylet() highlighter) and
 		# the mount/bridge script that wires it to the hidden #ls-editor-state textbox.
 		os.path.join(vendor, 'lyl-editor.bundle.js'),
 		os.path.join(WEB_DIR, 'lyl-editor-mount.js'),
 	]
-	tags = ['<script>window.__LILYSCRIPT_SOUNDFONT_URL=%r;</script>'
-		% (_file_url(os.path.join(WEB_DIR, 'soundfont')) + '/')]
+	tags = ['<script>window.__LILYSCRIPT_SOUNDFONT_URL=%r;window.__LILYSCRIPT_FLUID_URL=%r;</script>'
+		% (_file_url(os.path.join(WEB_DIR, 'soundfont')) + '/',
+		   _file_url(os.path.join(WEB_DIR, 'fluid')) + '/')]
 	tags.append('<link rel="stylesheet" href="%s">' % _file_url(os.path.join(WEB_DIR, 'score-player.css')))
 	tags.append('<link rel="stylesheet" href="%s">' % _file_url(os.path.join(WEB_DIR, 'lyl-editor.css')))
 	for s in scripts:

web/fluid-audio.js

@@ -0,0 +1,163 @@
+/* LilyScript FluidSynth audio backend — vanilla-JS port of lilylet-live-editor's
+ * src/lib/audio/fluidAudio.ts, exposed as window.LilyFluidAudio.
+ *
+ * Replaces the legacy single-piano MIDI.js soundfont with FluidSynth (js-synthesizer
+ * WASM) playing a General MIDI gm.sf3 (128 instruments), so each track sounds with
+ * its own program instead of everything-as-piano.
+ *
+ * Shaped to match the subset of music-widgets' MidiAudio API that score-player.js
+ * drives: empty / loadPlugin / resume / noteOn / noteOff / programChange /
+ * stopAllNotes. score-player.js sends noteOn/noteOff with an absolute
+ * performance.now()-based timestamp; we forward to FluidSynth's Sequencer via
+ * sendEventAt(event, dtMs, isAbsolute=false) ("play in dtMs ms"), preserving the
+ * look-ahead timing without reconciling against the sequencer's own tick origin.
+ *
+ * Fallback: gm.sf3 is ~40 MB and takes a moment to fetch + decode. While it loads we
+ * fall back to the legacy MIDI.js WebAudio piano (the small, locally-bundled
+ * acoustic-grand-piano soundfont) so playback is audible immediately; once FluidSynth
+ * is ready we switch to it transparently.
+ *
+ * Globals consumed (loaded via <script src> in <head> by app.py build_head):
+ *   window.JSSynth                     js-synthesizer UMD { AudioWorkletNodeSynthesizer, ... }
+ *   window.musicWidgetsBrowser.MidiAudio   legacy MIDI.js WebAudio piano (fallback)
+ *   window.__LILYSCRIPT_FLUID_URL      base URL of web/fluid/ (libfluidsynth + worklet)
+ *   window.__LILYSCRIPT_SOUNDFONT_URL  base URL of web/soundfont/ (gm.sf3 + piano)
+ */
+(function () {
+	'use strict';
+
+	function log (msg) { console.log('[LilyFluid]', msg); }
+
+	// URLs (with trailing slash) injected by app.py; fall back to relative paths.
+	var FLUID_URL = (window.__LILYSCRIPT_FLUID_URL || './fluid/');
+	var SOUNDFONT_URL = (window.__LILYSCRIPT_SOUNDFONT_URL || './soundfont/');
+
+	// The with-libsndfile build is required to decode .sf3 (ogg-compressed) fonts;
+	// it also reads .sf2. Both modules load into the AudioWorklet scope.
+	var LIBFLUIDSYNTH_FILE = FLUID_URL + 'libfluidsynth-2.4.6-with-libsndfile.js';
+	var WORKLET_FILE = FLUID_URL + 'js-synthesizer.worklet.js';
+	var SOUNDFONT_FILE = SOUNDFONT_URL + 'gm.sf3';
+
+	var audioCtx = null;
+	var synth = null;
+	var seq = null;
+	var node = null;          // the synth's AudioNode (kept for diagnostics)
+	var loaded = false;
+	var loadingPromise = null;
+
+	// Legacy MIDI.js fallback (single-piano), used until FluidSynth finishes loading.
+	var legacy = null;
+	var legacyReady = false;
+
+	// Inject the legacy UMD MidiAudio (fallback). Must be called before loadPlugin().
+	function init (legacyMidiAudio) {
+		legacy = legacyMidiAudio || null;
+	}
+
+	// True once the (preferred) FluidSynth backend has finished loading.
+	function ready () { return loaded; }
+
+	// True while FluidSynth is still loading (the fallback may already be audible).
+	function loading () { return !loaded && !!loadingPromise; }
+
+	// Whether any backend can produce sound yet (FluidSynth or the legacy fallback).
+	function empty () { return !loaded && !legacyReady; }
+
+	function delayFromNow (timestamp) { return Math.max(0, timestamp - performance.now()); }
+
+	function loadPlugin () {
+		if (loaded) return Promise.resolve();
+		if (loadingPromise) return loadingPromise;
+
+		// Start the lightweight legacy piano fallback immediately so playback is
+		// audible while the large FluidSynth soundfont downloads. Non-fatal on error.
+		if (legacy && legacy.WebAudio && legacy.WebAudio.empty && legacy.WebAudio.empty()) {
+			legacy.loadPlugin({ soundfontUrl: SOUNDFONT_URL, api: 'webaudio' })
+				.then(function () { legacyReady = true; log('legacy piano fallback ready'); })
+				.catch(function (err) { console.warn('[LilyFluid] legacy fallback failed:', err); });
+		} else if (legacy) {
+			legacyReady = true;
+		}
+
+		loadingPromise = (async function () {
+			audioCtx = new (window.AudioContext || window.webkitAudioContext)();
+			// libfluidsynth must be added before the worklet module.
+			await audioCtx.audioWorklet.addModule(LIBFLUIDSYNTH_FILE);
+			await audioCtx.audioWorklet.addModule(WORKLET_FILE);
+
+			synth = new window.JSSynth.AudioWorkletNodeSynthesizer();
+			synth.init(audioCtx.sampleRate);
+			// createAudioNode MUST be called before any other synth method.
+			node = synth.createAudioNode(audioCtx);
+			node.connect(audioCtx.destination);
+
+			var sfontBuffer = await (await fetch(SOUNDFONT_FILE)).arrayBuffer();
+			await synth.loadSFont(sfontBuffer);
+
+			seq = await synth.createSequencer();
+			await seq.registerSynthesizer(synth);
+			seq.setTimeScale(1000);   // 1 tick = 1 ms, matching performance.now()
+
+			// Hand off from the fallback: silence any lingering notes.
+			if (legacyReady && legacy && legacy.stopAllNotes) legacy.stopAllNotes();
+
+			loaded = true;
+			log('FluidSynth soundfont loaded (gm.sf3)');
+		})();
+		return loadingPromise;
+	}
+
+	// Resume audio output; must be called from a user gesture (autoplay policy).
+	async function resume () {
+		if (audioCtx && audioCtx.state === 'suspended') { try { await audioCtx.resume(); } catch (e) {} }
+		var WA = legacy && legacy.WebAudio;
+		if (legacyReady && WA && WA.needsWarmup && WA.needsWarmup() && WA.awaitWarmup) {
+			try { await WA.awaitWarmup(); } catch (e) {}
+		}
+	}
+
+	function noteOn (channel, note, velocity, timestamp) {
+		if (seq) { seq.sendEventAt({ type: 'noteon', channel: channel, key: note, vel: velocity }, delayFromNow(timestamp), false); return; }
+		if (legacyReady && legacy) legacy.noteOn(channel, note, velocity, timestamp);
+	}
+
+	function noteOff (channel, note, timestamp) {
+		if (seq) { seq.sendEventAt({ type: 'noteoff', channel: channel, key: note }, delayFromNow(timestamp), false); return; }
+		if (legacyReady && legacy) legacy.noteOff(channel, note, timestamp);
+	}
+
+	function programChange (channel, program) {
+		if (seq) { seq.sendEventAt({ type: 'programchange', channel: channel, preset: program }, 0, false); return; }
+		if (legacyReady && legacy) legacy.programChange(channel, program);
+	}
+
+	function stopAllNotes () {
+		if (seq) seq.removeAllEvents();   // drop the in-flight look-ahead window
+		if (synth) { for (var ch = 0; ch < 16; ++ch) synth.midiAllSoundsOff(ch); }
+		if (legacyReady && legacy && legacy.stopAllNotes) legacy.stopAllNotes();
+	}
+
+	window.LilyFluidAudio = {
+		init: init,
+		empty: empty,
+		ready: ready,
+		loading: loading,
+		loadPlugin: loadPlugin,
+		resume: resume,
+		noteOn: noteOn,
+		noteOff: noteOff,
+		programChange: programChange,
+		stopAllNotes: stopAllNotes,
+		// Lightweight diagnostics: backend/AudioContext state. (A scheduled note with
+		// no thrown error does NOT prove audio output — for that, tap node with an
+		// AnalyserNode in the console.) Handy when troubleshooting "no sound".
+		_debug: function () {
+			return {
+				loaded: loaded, legacyReady: legacyReady,
+				ctxState: audioCtx ? audioCtx.state : 'no-ctx',
+				hasSeq: !!seq, hasSynth: !!synth, hasNode: !!node,
+			};
+		},
+	};
+	log('fluid-audio.js loaded');
+})();

web/fluid/js-synthesizer.worklet.js

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93ba8c10322273523fa7de8c4b91d402692db8f24a52d7ca3bd31b99b5d06510
+size 29391

web/fluid/libfluidsynth-2.4.6-with-libsndfile.js

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c108b7f00ed348892d46f9e6236618d76ae7f3d2b6cb860b2990c0c2ce9dc67c
+size 2371527

web/score-player.css

@@ -146,3 +146,41 @@
 	background: #7c5cff;
 	transition: width 0.1s linear;
 }
+
+/* Sound-library status badge in the transport bar. While the GM soundfont loads:
+   dimmed + grayscaled + pulsing 🎹 (the small grand-piano fallback is audible).
+   Once ready: full-color 🎹 with a green ✓, shown 2s then faded out (hover reveals). */
+#ls-score .ls-sf {
+	position: relative;
+	font-size: 14px;
+	line-height: 1;
+	margin-left: 2px;
+	flex: 0 0 auto;
+	cursor: default;
+	opacity: 0.45;
+	filter: grayscale(1);
+	animation: ls-sf-pulse 1.4s ease-in-out infinite;
+}
+#ls-score .ls-sf.ready {
+	opacity: 1;
+	filter: none;
+	animation: none;
+}
+@keyframes ls-sf-pulse {
+	0%, 100% { opacity: 0.30; }
+	50%      { opacity: 0.65; }
+}
+/* green ✓ badge superscript on the ready piano */
+#ls-score .ls-sf .ls-sf-check {
+	display: none;
+	position: absolute;
+	top: -0.4em;
+	right: -0.7em;
+	font-size: 0.85em;
+	font-weight: 700;
+	color: #3fb950;
+	text-shadow: 0 0 2px rgba(0, 0, 0, 0.4);
+}
+#ls-score .ls-sf.ready .ls-sf-check {
+	display: inline;
+}

web/score-player.js

@@ -158,15 +158,36 @@
 		if (state.audioReady) return true;
 		const mw = window.musicWidgetsBrowser;
 		if (!mw) { log('musicWidgetsBrowser missing'); return false; }
-		state.audio = { MIDI: mw.MIDI, MidiPlayer: mw.MidiPlayer, MusicNotation: mw.MusicNotation, MidiAudio: mw.MidiAudio };
+		// MIDI/MidiPlayer/MusicNotation come from music-widgets; the *audio backend* is
+		// FluidSynth (js-synthesizer + gm.sf3, 128 GM instruments) via LilyFluidAudio,
+		// which falls back to music-widgets' single-piano MidiAudio while the large GM
+		// soundfont downloads. score-player drives MidiAudio.{noteOn,noteOff,
+		// programChange,stopAllNotes,resume}, all forwarded by LilyFluidAudio.
+		var backend = window.LilyFluidAudio;
+		if (backend && backend.init) {
+			backend.init(mw.MidiAudio);                 // inject the legacy piano fallback
+		} else {
+			log('LilyFluidAudio missing; using music-widgets MidiAudio directly');
+			backend = mw.MidiAudio;
+		}
+		state.audio = { MIDI: mw.MIDI, MidiPlayer: mw.MidiPlayer, MusicNotation: mw.MusicNotation, MidiAudio: backend };
+		startSfStatus();   // show the sound-library loading badge while the GM font downloads
 		try {
-			await state.audio.MidiAudio.loadPlugin({ soundfontUrl: SOUNDFONT_URL, api: 'webaudio' });
+			// Kick off loading; resolves only when the full GM soundfont is decoded.
+			// Don't block playback on it — the legacy piano fallback becomes audible
+			// much earlier, so report ready as soon as any backend can sound.
+			var fullLoad = backend.loadPlugin ? backend.loadPlugin({ soundfontUrl: SOUNDFONT_URL, api: 'webaudio' }) : Promise.resolve();
+			if (backend.empty) {
+				for (var i = 0; i < 100 && backend.empty(); i++) { await new Promise(function (r) { setTimeout(r, 100); }); }
+			} else {
+				await fullLoad;
+			}
+			if (backend.empty && backend.empty()) { log('audio not ready (no backend audible)'); return false; }
 			state.audioReady = true;
-			// warm up the AudioContext eagerly so later plays are already hot (the
-			// in-gesture warmup in play() is what actually satisfies autoplay policy)
-			var WA = state.audio.MidiAudio.WebAudio;
-			if (WA && WA.awaitWarmup) { try { await WA.awaitWarmup(); } catch (e) {} }
-			log('audio ready');
+			// warm up eagerly (the in-gesture resume() in play() is what actually
+			// satisfies the browser autoplay policy).
+			if (backend.resume) { try { await backend.resume(); } catch (e) {} }
+			log('audio ready (fluid=' + (backend.ready ? backend.ready() : '?') + ')');
 			return true;
 		} catch (e) {
 			log('audio load failed: ' + (e && e.message ? e.message : e));
@@ -209,10 +230,13 @@
 
 	async function play () {
 		if (!state.player || !state.midiData || state.isPlaying || state.generating) return;
-		// warm up the AudioContext (browser autoplay policy) inside the play gesture
-		// before scheduling notes, so the opening notes aren't dropped on first play
-		var WA = state.audio && state.audio.MidiAudio && state.audio.MidiAudio.WebAudio;
-		if (WA && WA.awaitWarmup) { try { await WA.awaitWarmup(); } catch (e) {} }
+		// warm up / resume the audio backend (browser autoplay policy) inside the play
+		// gesture before scheduling notes, so the opening notes aren't dropped on first
+		// play. LilyFluidAudio.resume() resumes the FluidSynth AudioContext (and warms
+		// the legacy fallback); fall back to the old WebAudio.awaitWarmup path.
+		var A0 = state.audio && state.audio.MidiAudio;
+		if (A0 && A0.resume) { try { await A0.resume(); } catch (e) {} }
+		else { var WA = A0 && A0.WebAudio; if (WA && WA.awaitWarmup) { try { await WA.awaitWarmup(); } catch (e) {} } }
 		state.isPlaying = true;
 		if (state.pausedTime > 0) {
 			state.playStartTime = performance.now() - state.pausedTime;
@@ -368,6 +392,28 @@
 		els.pauseBtn.style.display = state.isPlaying ? '' : 'none';
 	}
 
+	// ---- sound-library (soundfont) status badge -----------------------------
+	// Mirrors live-editor's Preview.svelte sf-status: pulsing/grayscale 🎹 while the
+	// GM soundfont loads (grand-piano fallback active), then a steady green 🎹✓ once
+	// ready (left in place, no fade). FluidSynth-only; if the backend lacks the
+	// loading/ready introspection (plain music-widgets MidiAudio) the badge stays hidden.
+	function startSfStatus () {
+		var F = state.audio && state.audio.MidiAudio;
+		if (!els.sf || !F || !F.ready || !F.loading) return;   // not the fluid backend
+		if (state._sfInterval) return;                          // already tracking
+		els.sf.style.display = '';
+		els.sf.classList.remove('ready');
+		function markReady () {
+			els.sf.classList.add('ready');
+			els.sf.title = 'Sound library ready — full instrument timbres';
+		}
+		if (F.ready()) { markReady(); return; }   // fast path (cached)
+		var iv = setInterval(function () {
+			if (F.ready()) { clearInterval(iv); state._sfInterval = null; markReady(); }
+		}, 200);
+		state._sfInterval = iv;
+	}
+
 	// Show the player bar only when: not generating, a score is rendered, audio
 	// available. While generating we keep SVG visible but hide the transport.
 	function updatePlayerVisibility () {
@@ -401,7 +447,10 @@
 			'<button class="ls-btn ls-pause" title="Pause" style="display:none">⏸</button>' +
 			'<button class="ls-btn ls-stop" title="Stop">■</button>' +
 			'<span class="ls-time">0:00 / 0:00</span>' +
-			'<div class="ls-progress"><div class="ls-fill"></div></div>';
+			'<div class="ls-progress"><div class="ls-fill"></div></div>' +
+			// sound-library status: pulsing 🎹 while the GM soundfont loads (piano
+			// fallback audible), green 🎹✓ once ready (shown 2s, then faded; hover reveals).
+			'<span class="ls-sf" style="display:none" title="Loading sound library… (grand-piano fallback active)">🎹<span class="ls-sf-check">✓</span></span>';
 
 		root.appendChild(player); root.appendChild(wrap);
 
@@ -412,6 +461,7 @@
 		els.time = player.querySelector('.ls-time');
 		els.progress = player.querySelector('.ls-progress');
 		els.fill = player.querySelector('.ls-fill');
+		els.sf = player.querySelector('.ls-sf');
 
 		els.playBtn.addEventListener('click', play);
 		els.pauseBtn.addEventListener('click', pause);

web/soundfont/gm.sf3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b85b6c2c61d10b2b91cddd41efcce7b25cd31c8271d511c73afafbef20b6fa3
+size 39900972

web/vendor/js-synthesizer.min.js

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:296f3cf76304b3650d7fb24a00263e2225e21d612fc0882234de7b1090da2b53
+size 32516