// "Show thinker": the live previews of the ComfyUI nodes (agate-comfyui nodes.py, _progress, live_preview // "side_by_side"), computed the same way so the two frontends show the same thing: // * the plan: the thinker's output for the conditional branch, 640 channels on a 16 x 16 grid. Its cells are // projected on the top 3 principal components (fitted once, at the first step, centred over the cells; each // component's sign fixed so its largest-magnitude loading is positive) and mapped to RGB with that step's 2% / // 98% quantiles -- a fixed basis, so a colour keeps its meaning while the plan evolves; // * the image the model expects at the end, x1 = z + (1 - t) v (guided velocity), shown with ComfyUI's SD 1.5 // latent -> RGB factors (its "latent2rgb" preview). const RGB_FACTORS = [[0.3512, 0.2297, 0.3227], [0.3250, 0.4974, 0.2350], [-0.2829, 0.1762, 0.2721], [-0.2120, -0.2616, -0.7177]]; // top-k eigenvectors of the (d x d) Gram matrix X^T X of centred X (n x d), as torch.linalg.eigh gives them (sign // aside), via the smaller (n x n) X X^T: block (subspace) iteration with a 12-vector block + Rayleigh-Ritz, which // separates near-equal eigenvalues that plain power iteration mixes; then v = X^T u / |X^T u|. function jacobiEig(A, m) { // symmetric m x m (Float64Array) -> {vals, vecs (column-major rows)} const a = Float64Array.from(A), V = new Float64Array(m * m); for (let i = 0; i < m; i++) V[i * m + i] = 1; for (let sweep = 0; sweep < 60; sweep++) { let off = 0; for (let p = 0; p < m; p++) for (let q = p + 1; q < m; q++) off += a[p * m + q] ** 2; if (off < 1e-24) break; for (let p = 0; p < m; p++) for (let q = p + 1; q < m; q++) { const apq = a[p * m + q]; if (Math.abs(apq) < 1e-300) continue; const th = (a[q * m + q] - a[p * m + p]) / (2 * apq); const t = Math.sign(th || 1) / (Math.abs(th) + Math.sqrt(th * th + 1)), c = 1 / Math.sqrt(t * t + 1), s = t * c; for (let k = 0; k < m; k++) { const x = a[k * m + p], y = a[k * m + q]; a[k * m + p] = c * x - s * y; a[k * m + q] = s * x + c * y; } for (let k = 0; k < m; k++) { const x = a[p * m + k], y = a[q * m + k]; a[p * m + k] = c * x - s * y; a[q * m + k] = s * x + c * y; } for (let k = 0; k < m; k++) { const x = V[k * m + p], y = V[k * m + q]; V[k * m + p] = c * x - s * y; V[k * m + q] = s * x + c * y; } } } return { vals: Array.from({ length: m }, (_, i) => a[i * m + i]), V }; } function topComponents(X, n, d, k = 3, iters = 60, block = 12) { const G = new Float64Array(n * n); for (let i = 0; i < n; i++) for (let j = i; j < n; j++) { let s = 0; const ri = i * d, rj = j * d; for (let a = 0; a < d; a++) s += X[ri + a] * X[rj + a]; G[i * n + j] = G[j * n + i] = s; } const m = Math.min(block, n); let Q = Array.from({ length: m }, (_, c) => Float64Array.from({ length: n }, (_, i) => Math.sin(1.7 * i + 0.9 * c + 1) + 0.01 * c)); const orth = (B) => { // modified Gram-Schmidt, in place for (let c = 0; c < B.length; c++) { for (let e = 0; e < c; e++) { let p = 0; for (let i = 0; i < n; i++) p += B[c][i] * B[e][i]; for (let i = 0; i < n; i++) B[c][i] -= p * B[e][i]; } let nrm = 0; for (let i = 0; i < n; i++) nrm += B[c][i] ** 2; nrm = Math.sqrt(nrm) || 1; for (let i = 0; i < n; i++) B[c][i] /= nrm; } return B; }; const mul = (q) => { const w = new Float64Array(n); for (let i = 0; i < n; i++) { let s = 0; const gi = i * n; for (let j = 0; j < n; j++) s += G[gi + j] * q[j]; w[i] = s; } return w; }; orth(Q); for (let it = 0; it < iters; it++) Q = orth(Q.map(mul)); const GQ = Q.map(mul), T = new Float64Array(m * m); // Rayleigh-Ritz on the block for (let a = 0; a < m; a++) for (let b = 0; b < m; b++) { let s = 0; for (let i = 0; i < n; i++) s += Q[a][i] * GQ[b][i]; T[a * m + b] = s; } for (let a = 0; a < m; a++) for (let b = 0; b < a; b++) T[a * m + b] = T[b * m + a] = (T[a * m + b] + T[b * m + a]) / 2; const { vals, V } = jacobiEig(T, m); const order = vals.map((v, i) => [v, i]).sort((x, y) => y[0] - x[0]).slice(0, k).map((x) => x[1]); return order.map((col) => { const u = new Float64Array(n); for (let a = 0; a < m; a++) { const w = V[a * m + col]; for (let i = 0; i < n; i++) u[i] += w * Q[a][i]; } const v = new Float64Array(d); for (let i = 0; i < n; i++) { const ui = u[i], ri = i * d; if (ui) for (let a = 0; a < d; a++) v[a] += ui * X[ri + a]; } let nrm = 0; for (let a = 0; a < d; a++) nrm += v[a] * v[a]; nrm = Math.sqrt(nrm) || 1; let piv = 0; for (let a = 0; a < d; a++) { v[a] /= nrm; if (Math.abs(v[a]) > Math.abs(v[piv])) piv = a; } if (v[piv] < 0) for (let a = 0; a < d; a++) v[a] = -v[a]; return v; }); } function quantile(sorted, q) { // torch.quantile, linear interpolation const pos = (sorted.length - 1) * q, lo = Math.floor(pos), hi = Math.ceil(pos); return sorted[lo] + (sorted[hi] - sorted[lo]) * (pos - lo); } // state: { basis, lo, hi } (filled at the first call of a generation). -> RGBA Uint8ClampedArray (g * g * 4) export function planPixels(plan, state, C = 640, g = 16) { const n = g * g, X = new Float64Array(n * C); for (let c = 0; c < C; c++) for (let p = 0; p < n; p++) X[p * C + c] = plan[c * n + p]; for (let c = 0; c < C; c++) { let m = 0; for (let p = 0; p < n; p++) m += X[p * C + c]; m /= n; for (let p = 0; p < n; p++) X[p * C + c] -= m; } if (!state.basis) state.basis = topComponents(X, n, C); const Y = state.basis.map((v) => { const y = new Float64Array(n); for (let p = 0; p < n; p++) { let s = 0; const r = p * C; for (let c = 0; c < C; c++) s += X[r + c] * v[c]; y[p] = s; } return y; }); if (!state.lo) { state.lo = Y.map((y) => quantile(Float64Array.from(y).sort(), 0.02)); state.hi = Y.map((y) => quantile(Float64Array.from(y).sort(), 0.98)); } const px = new Uint8ClampedArray(n * 4); for (let p = 0; p < n; p++) { for (let k = 0; k < 3; k++) px[p * 4 + k] = Math.round(Math.min(1, Math.max(0, (Y[k][p] - state.lo[k]) / (state.hi[k] - state.lo[k] + 1e-8))) * 255); px[p * 4 + 3] = 255; } return px; } // x1: Float32Array (4 * hw * hw) in the model's (scaled SD-VAE) latent space -> RGBA (hw * hw * 4) export function predPixels(x1, hw) { const n = hw * hw, px = new Uint8ClampedArray(n * 4); for (let p = 0; p < n; p++) { for (let k = 0; k < 3; k++) { let s = 0; for (let c = 0; c < 4; c++) s += x1[c * n + p] * RGB_FACTORS[c][k]; px[p * 4 + k] = Math.floor(Math.min(1, Math.max(0, (s + 1) / 2)) * 255); } px[p * 4 + 3] = 255; } return px; } // Draws previews; the colouring runs in a Web Worker (thinker-worker.js) when available, so the page and the // sampler never wait for it. A frame arriving while the previous one is still being coloured is dropped. export class ThinkerView { constructor(planCanvas, predCanvas, onDrawn = () => {}) { this.planCanvas = planCanvas; this.predCanvas = predCanvas; this.onDrawn = onDrawn; this.gen = 0; this.busy = false; this.dropped = 0; this.drawn = 0; try { this.worker = new Worker(new URL("./thinker-worker.js", import.meta.url), { type: "module" }); this.worker.onmessage = (e) => this._paint(e.data); } catch { this.worker = null; } this.reset(); } reset() { this.gen++; this.state = {}; this.busy = false; this.dropped = 0; this.drawn = 0; } get pending() { return this.busy; } // plan: Float32Array (640 * 16 * 16), x1: Float32Array (4 * hw * hw); both are transferred to the worker submit(plan, x1, hw, meta = {}) { if (this.busy) { this.dropped++; return false; } this.busy = true; const job = { gen: this.gen, plan, x1, hw, meta }; if (this.worker) this.worker.postMessage(job, [plan.buffer, x1.buffer]); else setTimeout(() => this._paint({ gen: job.gen, hw, meta, plan: planPixels(plan, this.state), pred: predPixels(x1, hw) }), 0); return true; } _paint({ gen, plan, pred, hw, meta, ms }) { this.busy = false; if (gen !== this.gen) return; const put = (cv, px, s) => { cv.width = s; cv.height = s; cv.getContext("2d").putImageData(new ImageData(px, s, s), 0, 0); }; put(this.planCanvas, plan, 16); put(this.predCanvas, pred, hw); this.drawn++; this.onDrawn({ ...meta, workerMs: ms, dropped: this.dropped, drawn: this.drawn }); } // synchronous path (tests): colour on this thread drawPlan(plan) { const px = planPixels(plan, this.state); const cv = this.planCanvas; cv.width = 16; cv.height = 16; cv.getContext("2d").putImageData(new ImageData(px, 16, 16), 0, 0); } }