| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="UTF-8"> |
| <title>Video user study — side-by-side rating</title> |
| <style> |
| :root { |
| --bg: #0f1115; |
| --panel: #161922; |
| --panel-2: #1c2030; |
| --panel-3: #232839; |
| --border: #2a2f42; |
| --border-strong: #3a4060; |
| --fg: #e6e8ef; |
| --muted: #8b91a8; |
| --muted-2: #5e6480; |
| --accent: #6aa9ff; |
| --accent-strong: #8ec0ff; |
| --good: #28c76f; |
| --warn: #ff9f43; |
| --bad: #ea5455; |
| --tag: #232a3d; |
| --score-1: #ea5455; |
| --score-2: #ff7d52; |
| --score-3: #ffa84a; |
| --score-4: #8acf6d; |
| --score-5: #28c76f; |
| } |
| * { box-sizing: border-box; } |
| html, body { margin: 0; padding: 0; background: var(--bg); color: var(--fg); |
| font-family: -apple-system, BlinkMacSystemFont, "Inter", "Helvetica Neue", |
| "PingFang SC", "Microsoft YaHei", sans-serif; |
| height: 100%; font-size: 14px; line-height: 1.5; } |
| #app { display: flex; flex-direction: column; height: 100vh; } |
| |
| |
| header.top { padding: 10px 18px; border-bottom: 1px solid var(--border); |
| background: var(--panel); display: flex; gap: 16px; align-items: center; |
| flex-wrap: wrap; } |
| .title { font-weight: 600; font-size: 15px; } |
| .subtitle { color: var(--muted); font-size: 12px; } |
| .progress { margin-left: auto; display: flex; gap: 14px; align-items: center; } |
| .progress-bar { width: 200px; height: 6px; background: var(--panel-3); |
| border-radius: 3px; overflow: hidden; } |
| .progress-fill { height: 100%; background: linear-gradient(90deg, var(--accent), var(--good)); |
| width: 0%; transition: width 0.3s; } |
| .progress-text { font-size: 12px; color: var(--muted); white-space: nowrap; } |
| .progress-text .num { color: var(--fg); font-weight: 600; } |
| |
| |
| .toolbar { padding: 8px 18px; background: var(--panel); border-bottom: 1px solid var(--border); |
| display: flex; gap: 8px; align-items: center; flex-wrap: wrap; } |
| .btn { background: var(--panel-3); border: 1px solid var(--border-strong); |
| color: var(--fg); padding: 5px 12px; border-radius: 4px; font-size: 12px; |
| cursor: pointer; font-family: inherit; transition: background 0.15s; } |
| .btn:hover { background: #2c3450; } |
| .btn.primary { background: var(--accent); color: #0a1a2a; border-color: var(--accent); font-weight: 600; } |
| .btn.primary:hover { background: var(--accent-strong); } |
| .btn.ghost { background: transparent; border-color: var(--border); color: var(--muted); } |
| .btn.ghost:hover { color: var(--fg); border-color: var(--border-strong); } |
| .toolbar .sep { width: 1px; height: 18px; background: var(--border); margin: 0 4px; } |
| .rater-box { display: flex; gap: 6px; align-items: center; font-size: 12px; color: var(--muted); } |
| .rater-box input { background: var(--panel-3); border: 1px solid var(--border); |
| color: var(--fg); padding: 4px 8px; border-radius: 3px; font-family: inherit; |
| font-size: 12px; width: 140px; } |
| |
| |
| main { flex: 1; display: flex; min-height: 0; } |
| aside.list { width: 240px; border-right: 1px solid var(--border); background: var(--panel); |
| overflow-y: auto; flex-shrink: 0; } |
| .list-header { padding: 10px 14px; font-size: 11px; color: var(--muted); |
| text-transform: uppercase; letter-spacing: 0.5px; border-bottom: 1px solid var(--border); |
| position: sticky; top: 0; background: var(--panel); } |
| .case-row { padding: 9px 14px; border-bottom: 1px solid var(--border); cursor: pointer; |
| display: flex; align-items: center; gap: 8px; font-size: 13px; } |
| .case-row:hover { background: var(--panel-2); } |
| .case-row.active { background: var(--panel-2); border-left: 3px solid var(--accent); |
| padding-left: 11px; } |
| .case-row .id { font-family: ui-monospace, "SF Mono", Menlo, monospace; flex: 1; } |
| .case-row .status-dot { width: 8px; height: 8px; border-radius: 50%; flex-shrink: 0; |
| background: var(--muted-2); } |
| .case-row.partial .status-dot { background: var(--warn); } |
| .case-row.complete .status-dot { background: var(--good); } |
| |
| section.detail { flex: 1; overflow-y: auto; padding: 18px 24px 80px; min-width: 0; } |
| |
| |
| .instructions { background: var(--panel); border: 1px solid var(--border); border-radius: 8px; |
| padding: 14px 18px; margin-bottom: 18px; } |
| .instructions summary { font-weight: 600; cursor: pointer; user-select: none; |
| list-style: none; display: flex; align-items: center; gap: 8px; } |
| .instructions summary::before { content: '▸'; color: var(--muted); font-size: 11px; } |
| .instructions[open] summary::before { content: '▾'; } |
| .instructions h4 { font-size: 13px; color: var(--accent-strong); |
| margin: 14px 0 6px; text-transform: uppercase; letter-spacing: 0.4px; } |
| .instructions ul { margin: 4px 0 4px 18px; padding: 0; color: var(--muted); } |
| .instructions li { margin: 3px 0; } |
| .instructions p { color: var(--muted); margin: 6px 0; } |
| .instructions code { background: var(--panel-3); padding: 1px 5px; border-radius: 3px; |
| font-size: 12px; color: var(--accent-strong); } |
| .scale-row { display: flex; gap: 6px; flex-wrap: wrap; margin-top: 6px; } |
| .scale-row .scale-pill { padding: 3px 10px; border-radius: 12px; font-size: 11px; |
| font-weight: 600; color: #0a1a0a; } |
| |
| |
| .case-header { display: flex; gap: 16px; align-items: flex-start; margin-bottom: 14px; |
| flex-wrap: wrap; } |
| .case-header .case-id { font-family: ui-monospace, monospace; font-size: 18px; |
| font-weight: 600; } |
| .case-header .case-counter { color: var(--muted); font-size: 12px; } |
| .prompt-box { background: var(--panel); border: 1px solid var(--border); border-radius: 6px; |
| padding: 12px 14px; line-height: 1.6; font-size: 14px; flex: 1; min-width: 280px; } |
| .prompt-box .label { font-size: 10px; text-transform: uppercase; color: var(--muted); |
| letter-spacing: 0.5px; margin-bottom: 4px; } |
| .ref-img { max-height: 140px; max-width: 220px; border-radius: 6px; |
| border: 1px solid var(--border); display: block; } |
| |
| |
| .video-grid { display: grid; gap: 12px; margin-bottom: 14px; } |
| .video-grid.cols-2 { grid-template-columns: 1fr 1fr; } |
| .video-grid.cols-3 { grid-template-columns: 1fr 1fr 1fr; } |
| .video-grid.cols-4 { grid-template-columns: 1fr 1fr 1fr 1fr; } |
| @media (max-width: 1100px) { |
| .video-grid.cols-3, .video-grid.cols-4 { grid-template-columns: 1fr 1fr; } |
| } |
| .video-tile { display: flex; flex-direction: column; gap: 6px; } |
| .video-tile .slot-label { font-size: 12px; color: var(--muted); |
| font-family: ui-monospace, monospace; display: flex; align-items: center; gap: 8px; } |
| .video-tile .slot-label .slot-num { padding: 2px 8px; background: var(--panel-3); |
| border-radius: 3px; color: var(--fg); font-weight: 600; } |
| .video-frame { position: relative; background: #000; border: 1px solid var(--border); |
| border-radius: 6px; overflow: hidden; aspect-ratio: 16 / 9; } |
| .video-frame video { width: 100%; height: 100%; display: block; background: #000; } |
| .video-frame .empty-state { position: absolute; inset: 0; display: flex; |
| flex-direction: column; align-items: center; justify-content: center; |
| background: repeating-linear-gradient(45deg, #1a1d28 0 10px, #16191f 10px 20px); |
| color: var(--muted); font-size: 12px; gap: 6px; padding: 12px; text-align: center; } |
| .video-frame .empty-state .icon { font-size: 28px; opacity: 0.3; } |
| .video-frame .empty-state .path { font-family: ui-monospace, monospace; font-size: 10px; |
| color: var(--muted-2); word-break: break-all; max-width: 90%; } |
| |
| |
| .video-controls { background: var(--panel); border: 1px solid var(--border); |
| border-radius: 6px; padding: 8px 12px; display: flex; gap: 10px; |
| align-items: center; flex-wrap: wrap; margin-bottom: 14px; font-size: 12px; |
| color: var(--muted); } |
| .video-controls .ctl-btn { background: var(--panel-3); border: 1px solid var(--border-strong); |
| color: var(--fg); padding: 4px 10px; border-radius: 3px; font-size: 12px; |
| cursor: pointer; font-family: inherit; } |
| .video-controls .ctl-btn:hover { background: #2c3450; } |
| .video-controls .ctl-btn.on { background: var(--accent); color: #0a1a2a; border-color: var(--accent); } |
| .video-controls select { background: var(--panel-3); border: 1px solid var(--border); |
| color: var(--fg); padding: 3px 6px; border-radius: 3px; font-size: 12px; font-family: inherit; } |
| |
| |
| .ratings-table { background: var(--panel); border: 1px solid var(--border); |
| border-radius: 8px; overflow: hidden; } |
| .ratings-table table { width: 100%; border-collapse: collapse; } |
| .ratings-table th, .ratings-table td { padding: 8px 10px; text-align: left; |
| border-bottom: 1px solid var(--border); font-size: 13px; vertical-align: middle; } |
| .ratings-table thead th { background: var(--panel-2); color: var(--muted); |
| font-size: 11px; text-transform: uppercase; letter-spacing: 0.5px; |
| font-weight: 600; position: sticky; top: 0; } |
| .ratings-table thead th.q-col { width: 38%; } |
| .ratings-table thead th.video-col { text-align: center; } |
| .dim-row td { background: var(--panel-2); padding-top: 10px; padding-bottom: 6px; |
| font-weight: 600; font-size: 12px; text-transform: uppercase; |
| letter-spacing: 0.5px; color: var(--accent-strong); border-bottom: 1px solid var(--border-strong); } |
| .dim-row .dim-hint { font-weight: 400; color: var(--muted); margin-left: 8px; |
| font-size: 11px; text-transform: none; letter-spacing: 0; } |
| .q-cell { color: var(--fg); } |
| .q-cell .q-name { font-weight: 500; } |
| .q-cell .q-desc { color: var(--muted); font-size: 12px; margin-top: 2px; } |
| .rate-cell { text-align: center; min-width: 200px; } |
| .likert { display: inline-flex; gap: 3px; } |
| .likert button { width: 32px; height: 32px; border: 1px solid var(--border); |
| background: var(--panel-3); color: var(--fg); border-radius: 4px; |
| cursor: pointer; font-family: inherit; font-size: 13px; font-weight: 500; |
| transition: transform 0.08s, border-color 0.15s; } |
| .likert button:hover { border-color: var(--accent); transform: translateY(-1px); } |
| .likert button.selected { color: #0a1a0a; font-weight: 700; |
| border-color: rgba(255,255,255,0.2); } |
| .likert button.selected[data-score="1"] { background: var(--score-1); } |
| .likert button.selected[data-score="2"] { background: var(--score-2); } |
| .likert button.selected[data-score="3"] { background: var(--score-3); } |
| .likert button.selected[data-score="4"] { background: var(--score-4); } |
| .likert button.selected[data-score="5"] { background: var(--score-5); } |
| |
| |
| footer.bottom { position: fixed; bottom: 0; left: 240px; right: 0; |
| background: var(--panel); border-top: 1px solid var(--border); |
| padding: 10px 24px; display: flex; gap: 12px; align-items: center; |
| z-index: 10; } |
| footer.bottom .summary { color: var(--muted); font-size: 12px; flex: 1; } |
| footer.bottom .summary .num { color: var(--fg); font-weight: 600; } |
| footer.bottom .nav-btns { display: flex; gap: 8px; } |
| |
| |
| .empty-app { padding: 40px; max-width: 760px; margin: 0 auto; } |
| .empty-app h2 { color: var(--fg); margin-bottom: 8px; } |
| .empty-app p { color: var(--muted); margin: 8px 0; line-height: 1.7; } |
| .empty-app pre { background: var(--panel); border: 1px solid var(--border); |
| border-radius: 6px; padding: 14px; font-size: 12px; color: var(--accent-strong); |
| overflow-x: auto; line-height: 1.5; } |
| .empty-app code { background: var(--panel-3); padding: 1px 5px; border-radius: 3px; |
| font-size: 12px; color: var(--accent-strong); } |
| |
| |
| .toast { position: fixed; bottom: 70px; right: 20px; background: var(--good); |
| color: #0a1a0a; padding: 10px 16px; border-radius: 6px; font-size: 13px; |
| font-weight: 600; opacity: 0; transition: opacity 0.2s; z-index: 100; |
| pointer-events: none; } |
| |
| .sync-pill { padding: 4px 10px; border-radius: 12px; font-size: 11px; |
| font-family: ui-monospace, monospace; background: var(--panel-3); color: var(--muted); |
| border: 1px solid var(--border); white-space: nowrap; cursor: help; } |
| .sync-pill[data-state="syncing"] { background: #1f2a44; color: var(--accent); border-color: #3a4a78; } |
| .sync-pill[data-state="synced"] { background: #1c3422; color: var(--good); border-color: #2a5a37; } |
| .sync-pill[data-state="local-only"] { background: #3a3520; color: var(--warn); border-color: #5a4a25; } |
| .sync-pill[data-state="error"] { background: #3a2020; color: var(--bad); border-color: #5a2828; } |
| .toast.show { opacity: 1; } |
| |
| kbd { background: var(--panel-3); border: 1px solid var(--border-strong); |
| border-radius: 3px; padding: 1px 5px; font-family: ui-monospace, monospace; |
| font-size: 11px; color: var(--fg); } |
| </style> |
| </head> |
| <body> |
| <div id="app"></div> |
| <div class="toast" id="toast"></div> |
|
|
| <script> |
| |
| |
| |
| const DIMENSIONS = [ |
| { |
| key: "instruction_following", |
| name: "Instruction following", |
| hint: "Does the video do what the prompt asked?", |
| questions: [ |
| { key: "controllability", |
| name: "Controllability", |
| desc: "Does the video respond to control signals (camera motion, conditioning, scene cues)?" }, |
| { key: "editing_correctness", |
| name: "Editing correctness", |
| desc: "Are requested edits applied without breaking unrelated content?" }, |
| { key: "prompt_adherence", |
| name: "Prompt adherence", |
| desc: "Does the content match what the prompt described (subjects, actions, setting)?" }, |
| ], |
| }, |
| { |
| key: "visual_quality", |
| name: "Visual quality", |
| hint: "How good does each frame look?", |
| questions: [ |
| { key: "aesthetics", |
| name: "Aesthetics", |
| desc: "Composition, color, lighting, overall pleasing look." }, |
| { key: "sharpness", |
| name: "Sharpness", |
| desc: "Clarity and detail; absence of blur or compression artifacts." }, |
| { key: "realism", |
| name: "Realism", |
| desc: "Photorealism or stylistic fidelity (no uncanny / broken artifacts)." }, |
| ], |
| }, |
| { |
| key: "temporal_consistency", |
| name: "Temporal consistency", |
| hint: "How stable is the content across frames?", |
| questions: [ |
| { key: "anti_flicker", |
| name: "Anti-flicker", |
| desc: "Absence of frame-to-frame brightness, color, or texture flicker." }, |
| { key: "person_stability", |
| name: "Person stability", |
| desc: "Faces, identities, and body proportions stay consistent across frames." }, |
| { key: "object_stability", |
| name: "Object stability", |
| desc: "Props, clothing, and objects keep their shape and position over time." }, |
| { key: "environment_stability", |
| name: "Environment stability", |
| desc: "Background, lighting, and scene layout stay coherent across frames." }, |
| ], |
| }, |
| { |
| key: "motion_realism", |
| name: "Motion realism", |
| hint: "Does motion behave plausibly?", |
| questions: [ |
| { key: "physics", |
| name: "Physics", |
| desc: "Gravity, collisions, fluid / cloth dynamics behave plausibly." }, |
| { key: "movement", |
| name: "Movement", |
| desc: "Body and object motion flows smoothly without warping or teleporting." }, |
| { key: "camera", |
| name: "Camera", |
| desc: "Camera movement (pan, dolly, focal change) feels natural and intentional." }, |
| ], |
| }, |
| ]; |
| |
| const SCALE_LABELS = { |
| 1: "Very poor", |
| 2: "Poor", |
| 3: "Fair", |
| 4: "Good", |
| 5: "Excellent", |
| }; |
| |
| const STORAGE_KEY = "user_study_video_ratings_v1"; |
| const TOTAL_QUESTIONS = DIMENSIONS.reduce((n, d) => n + d.questions.length, 0); |
| |
| |
| |
| |
| let CASES = []; |
| let SHUFFLE = {}; |
| let RATINGS = {}; |
| let RATER = { id: "", note: "", started_at: null }; |
| let currentIdx = 0; |
| let anonymize = true; |
| |
| |
| let SYNC_STATUS = "idle"; |
| let SYNC_TIMERS = {}; |
| let SYNC_ENDPOINT = "./submit"; |
| let SYNC_LAST_DETAIL = ""; |
| |
| |
| |
| |
| function loadState() { |
| try { |
| const raw = localStorage.getItem(STORAGE_KEY); |
| if (!raw) return null; |
| return JSON.parse(raw); |
| } catch (e) { return null; } |
| } |
| |
| function saveState() { |
| const blob = { |
| rater: RATER, |
| anonymize, |
| shuffle: SHUFFLE, |
| ratings: RATINGS, |
| saved_at: new Date().toISOString(), |
| }; |
| try { |
| localStorage.setItem(STORAGE_KEY, JSON.stringify(blob)); |
| } catch (e) { |
| showToast("⚠ failed to save: " + e.message, true); |
| } |
| } |
| |
| function genRaterId() { |
| return "rater_" + Math.random().toString(36).slice(2, 8) + |
| "_" + Date.now().toString(36).slice(-4); |
| } |
| |
| |
| |
| |
| function buildShuffle(cases) { |
| |
| for (const c of cases) { |
| if (SHUFFLE[c.id] && SHUFFLE[c.id].length === c.videos.length) continue; |
| const indices = c.videos.map((_, i) => i); |
| if (anonymize) { |
| |
| const seed = hashStr(RATER.id + ":" + c.id); |
| let s = seed; |
| for (let i = indices.length - 1; i > 0; i--) { |
| s = (s * 1664525 + 1013904223) >>> 0; |
| const j = s % (i + 1); |
| [indices[i], indices[j]] = [indices[j], indices[i]]; |
| } |
| } |
| SHUFFLE[c.id] = indices; |
| } |
| } |
| |
| function hashStr(s) { |
| let h = 2166136261 >>> 0; |
| for (let i = 0; i < s.length; i++) { |
| h ^= s.charCodeAt(i); |
| h = (h * 16777619) >>> 0; |
| } |
| return h; |
| } |
| |
| |
| |
| |
| function getCaseRatings(caseId) { |
| if (!RATINGS[caseId]) RATINGS[caseId] = {}; |
| return RATINGS[caseId]; |
| } |
| |
| function getSlotRatings(caseId, slotIdx) { |
| const m = getCaseRatings(caseId); |
| if (!m[slotIdx]) m[slotIdx] = {}; |
| return m[slotIdx]; |
| } |
| |
| function setRating(caseId, slotIdx, qkey, score) { |
| const r = getSlotRatings(caseId, slotIdx); |
| r[qkey] = score; |
| saveState(); |
| updateProgress(); |
| updateCaseStatus(caseId); |
| scheduleSubmit(caseId); |
| } |
| |
| |
| |
| |
| |
| |
| |
| function buildCaseSubmitPayload(caseId) { |
| const c = CASES.find(x => x.id === caseId); |
| if (!c) return null; |
| const slots = SHUFFLE[c.id] || c.videos.map((_, i) => i); |
| return { |
| rater_id: RATER.id || "anon", |
| case_id: c.id, |
| payload: { |
| rater: RATER, |
| anonymize_used: anonymize, |
| submitted_at: new Date().toISOString(), |
| prompt: c.prompt, |
| videos: slots.map((origIdx, slotIdx) => ({ |
| slot: String.fromCharCode(65 + slotIdx), |
| slot_index: slotIdx, |
| original_index: origIdx, |
| model: c.videos[origIdx].model, |
| path: c.videos[origIdx].path, |
| ratings: (RATINGS[c.id] || {})[slotIdx] || {}, |
| })), |
| }, |
| }; |
| } |
| |
| function scheduleSubmit(caseId, delayMs = 1500) { |
| |
| |
| if (SYNC_STATUS === "local-only") return; |
| if (SYNC_TIMERS[caseId]) clearTimeout(SYNC_TIMERS[caseId]); |
| SYNC_TIMERS[caseId] = setTimeout(() => { |
| delete SYNC_TIMERS[caseId]; |
| submitCaseToServer(caseId); |
| }, delayMs); |
| } |
| |
| async function submitCaseToServer(caseId) { |
| const body = buildCaseSubmitPayload(caseId); |
| if (!body) return; |
| setSyncStatus("syncing"); |
| try { |
| const resp = await fetch(SYNC_ENDPOINT, { |
| method: "POST", |
| headers: { "Content-Type": "application/json" }, |
| body: JSON.stringify(body), |
| }); |
| if (resp.ok) setSyncStatus("synced", `last write: ${body.case_id} @ ${new Date().toLocaleTimeString()}`); |
| else if (resp.status === 404 || resp.status === 503) |
| setSyncStatus("local-only", `server not configured (HTTP ${resp.status})`); |
| else setSyncStatus("error", `HTTP ${resp.status}`); |
| } catch (e) { |
| setSyncStatus("local-only", "no /submit endpoint reachable"); |
| } |
| } |
| |
| function setSyncStatus(state, detail) { |
| SYNC_STATUS = state; |
| if (detail) SYNC_LAST_DETAIL = detail; |
| const pill = document.getElementById("syncPill"); |
| if (!pill) return; |
| const label = { |
| idle: "💾 local", |
| syncing: "↻ syncing…", |
| synced: "✓ synced", |
| "local-only": "💾 local-only", |
| error: "✕ sync error", |
| }[state] || state; |
| pill.textContent = label; |
| pill.dataset.state = state; |
| pill.title = SYNC_LAST_DETAIL || "ratings autosave to your browser; ./submit forwards to the server when available"; |
| } |
| |
| async function probeSyncEndpoint() { |
| try { |
| const resp = await fetch(SYNC_ENDPOINT.replace(/\/submit$/, "/submit/health"), |
| { method: "GET", cache: "no-cache" }); |
| if (resp.ok) { |
| const j = await resp.json().catch(() => ({})); |
| setSyncStatus(j.configured ? "idle" : "local-only", |
| j.configured ? "server ready — ratings will sync to dataset" : (j.reason || "server not configured")); |
| } else { |
| setSyncStatus("local-only", `health probe HTTP ${resp.status}`); |
| } |
| } catch (e) { |
| setSyncStatus("local-only", "no /submit endpoint reachable"); |
| } |
| } |
| |
| function caseProgress(caseCase) { |
| const slots = SHUFFLE[caseCase.id] || []; |
| const expected = slots.length * TOTAL_QUESTIONS; |
| let done = 0; |
| for (let s = 0; s < slots.length; s++) { |
| const r = (RATINGS[caseCase.id] || {})[s] || {}; |
| done += Object.keys(r).length; |
| } |
| return { done, expected }; |
| } |
| |
| function totalProgress() { |
| let done = 0, expected = 0; |
| for (const c of CASES) { |
| const p = caseProgress(c); |
| done += p.done; expected += p.expected; |
| } |
| return { done, expected }; |
| } |
| |
| |
| |
| |
| function el(tag, attrs = {}, ...children) { |
| const e = document.createElement(tag); |
| for (const [k, v] of Object.entries(attrs)) { |
| if (k === "class") e.className = v; |
| else if (k === "html") e.innerHTML = v; |
| else if (k.startsWith("on") && typeof v === "function") { |
| e.addEventListener(k.slice(2).toLowerCase(), v); |
| } else if (v !== false && v != null) e.setAttribute(k, v); |
| } |
| for (const c of children) { |
| if (c == null || c === false) continue; |
| e.appendChild(typeof c === "string" ? document.createTextNode(c) : c); |
| } |
| return e; |
| } |
| |
| function escapeHtml(s) { |
| return String(s).replace(/[&<>"']/g, c => |
| ({'&':'&','<':'<','>':'>','"':'"',"'":'''})[c]); |
| } |
| |
| function showToast(msg, isError = false) { |
| const t = document.getElementById("toast"); |
| t.textContent = msg; |
| t.style.background = isError ? "var(--bad)" : "var(--good)"; |
| t.style.color = isError ? "#fff" : "#0a1a0a"; |
| t.classList.add("show"); |
| setTimeout(() => t.classList.remove("show"), 1800); |
| } |
| |
| |
| |
| |
| function renderShell() { |
| const app = document.getElementById("app"); |
| app.innerHTML = ""; |
| app.appendChild(renderHeader()); |
| app.appendChild(renderToolbar()); |
| const main = el("main"); |
| main.appendChild(renderSidebar()); |
| main.appendChild(el("section", { class: "detail", id: "detail" })); |
| app.appendChild(main); |
| app.appendChild(renderFooter()); |
| renderDetail(); |
| updateProgress(); |
| } |
| |
| function renderHeader() { |
| return el("header", { class: "top" }, |
| el("div", {}, |
| el("div", { class: "title" }, "Video user study — side-by-side rating"), |
| el("div", { class: "subtitle" }, |
| `${CASES.length} cases · 4 dimensions · ${TOTAL_QUESTIONS} sub-questions per video · 1–5 Likert`), |
| ), |
| el("div", { class: "progress" }, |
| el("div", { class: "progress-bar" }, |
| el("div", { class: "progress-fill", id: "progressFill" })), |
| el("span", { class: "progress-text", id: "progressText" }, "0 / 0"), |
| ), |
| ); |
| } |
| |
| function renderToolbar() { |
| const tb = el("div", { class: "toolbar" }); |
| |
| const raterInput = el("input", { |
| type: "text", |
| placeholder: "rater id", |
| value: RATER.id, |
| onchange: (e) => { |
| RATER.id = e.target.value.trim() || genRaterId(); |
| |
| SHUFFLE = {}; |
| buildShuffle(CASES); |
| saveState(); |
| renderDetail(); |
| }, |
| }); |
| |
| tb.appendChild(el("div", { class: "rater-box" }, |
| el("span", {}, "Rater:"), raterInput)); |
| tb.appendChild(el("div", { class: "sep" })); |
| |
| tb.appendChild(el("button", { |
| class: "btn ghost", |
| onclick: toggleAnon, |
| }, anonymize ? "🔒 Anonymized" : "👁 Reveal models")); |
| |
| tb.appendChild(el("div", { class: "sep" })); |
| |
| tb.appendChild(el("span", { |
| id: "syncPill", class: "sync-pill", |
| "data-state": SYNC_STATUS, |
| title: SYNC_LAST_DETAIL || "ratings autosave to your browser; ./submit forwards to the server when available", |
| }, "💾 local")); |
| |
| tb.appendChild(el("div", { class: "sep" })); |
| |
| tb.appendChild(el("button", { |
| class: "btn primary", |
| onclick: exportJSON, |
| }, "↓ Export JSON")); |
| |
| tb.appendChild(el("button", { |
| class: "btn ghost", |
| onclick: resetCurrentCase, |
| }, "↺ Reset this case")); |
| |
| tb.appendChild(el("button", { |
| class: "btn ghost", |
| onclick: resetAll, |
| }, "✕ Reset all")); |
| |
| return tb; |
| } |
| |
| function renderSidebar() { |
| const aside = el("aside", { class: "list" }); |
| aside.appendChild(el("div", { class: "list-header" }, `Cases (${CASES.length})`)); |
| CASES.forEach((c, i) => { |
| const p = caseProgress(c); |
| let cls = "case-row"; |
| if (i === currentIdx) cls += " active"; |
| if (p.done === 0) { } |
| else if (p.done < p.expected) cls += " partial"; |
| else cls += " complete"; |
| aside.appendChild(el("div", { |
| class: cls, |
| "data-case-id": c.id, |
| onclick: () => goTo(i), |
| }, |
| el("span", { class: "status-dot" }), |
| el("span", { class: "id" }, c.id), |
| el("span", { style: "font-size:11px;color:var(--muted)" }, |
| `${p.done}/${p.expected}`), |
| )); |
| }); |
| return aside; |
| } |
| |
| function renderFooter() { |
| return el("footer", { class: "bottom" }, |
| el("div", { class: "summary", id: "footerSummary" }), |
| el("div", { class: "nav-btns" }, |
| el("button", { class: "btn", onclick: () => step(-1) }, "← Prev"), |
| el("button", { class: "btn", onclick: () => step(1) }, "Next →"), |
| ), |
| ); |
| } |
| |
| |
| |
| |
| function renderDetail() { |
| const detail = document.getElementById("detail"); |
| if (!detail) return; |
| detail.innerHTML = ""; |
| if (!CASES.length) return; |
| |
| detail.appendChild(renderInstructions(currentIdx === 0)); |
| |
| const c = CASES[currentIdx]; |
| detail.appendChild(renderCaseHeader(c)); |
| detail.appendChild(renderVideoGrid(c)); |
| detail.appendChild(renderVideoControls(c)); |
| detail.appendChild(renderRatingsTable(c)); |
| |
| |
| document.querySelectorAll(".case-row").forEach((row, i) => { |
| row.classList.toggle("active", i === currentIdx); |
| }); |
| updateFooterSummary(); |
| } |
| |
| function renderInstructions(forceOpen) { |
| const det = el("details", { class: "instructions" }); |
| if (forceOpen) det.setAttribute("open", ""); |
| det.appendChild(el("summary", {}, |
| "How to rate (click to expand) — 4 dimensions, " + TOTAL_QUESTIONS + " sub-questions, 1–5 each")); |
| |
| const body = el("div"); |
| body.innerHTML = ` |
| <p>For each case, watch all videos side by side, then rate <strong>each video independently</strong> |
| on every sub-question. The rating table groups sub-questions by dimension; videos appear as columns.</p> |
| |
| <h4>Likert scale (1 = worst, 5 = best)</h4> |
| <div class="scale-row"> |
| <span class="scale-pill" style="background:var(--score-1)">1 · Very poor</span> |
| <span class="scale-pill" style="background:var(--score-2)">2 · Poor</span> |
| <span class="scale-pill" style="background:var(--score-3)">3 · Fair</span> |
| <span class="scale-pill" style="background:var(--score-4)">4 · Good</span> |
| <span class="scale-pill" style="background:var(--score-5)">5 · Excellent</span> |
| </div> |
| |
| <h4>Dimensions</h4> |
| <ul> |
| <li><strong>Instruction following</strong> — controllability, editing correctness, prompt adherence.</li> |
| <li><strong>Visual quality</strong> — aesthetics, sharpness, realism.</li> |
| <li><strong>Temporal consistency</strong> — anti-flicker, person / object / environment stability.</li> |
| <li><strong>Motion realism</strong> — physics, movement, camera motion.</li> |
| </ul> |
| |
| <h4>Tips</h4> |
| <ul> |
| <li>Use <kbd>Space</kbd> to play / pause all videos in sync, <kbd>R</kbd> to restart, <kbd>L</kbd> to toggle loop.</li> |
| <li>Use <kbd>←</kbd> / <kbd>→</kbd> (or <kbd>j</kbd> / <kbd>k</kbd>) to jump between cases.</li> |
| <li>Ratings autosave to your browser's localStorage. Click <code>↓ Export JSON</code> when finished.</li> |
| <li>Videos are <strong>anonymized by default</strong> — model identity is hidden until you toggle reveal.</li> |
| <li>If a tile shows "no video file", the manifest path doesn't exist on disk yet — drop a real <code>.mp4</code> at that path and refresh.</li> |
| </ul> |
| `; |
| det.appendChild(body); |
| return det; |
| } |
| |
| function renderCaseHeader(c) { |
| const wrap = el("div", { class: "case-header" }); |
| const left = el("div", { style: "flex:0 0 auto;display:flex;flex-direction:column;gap:4px" }, |
| el("span", { class: "case-id" }, c.id), |
| el("span", { class: "case-counter" }, |
| `Case ${currentIdx + 1} of ${CASES.length}`), |
| ); |
| wrap.appendChild(left); |
| |
| const promptBox = el("div", { class: "prompt-box" }); |
| promptBox.appendChild(el("div", { class: "label" }, "Prompt / instruction")); |
| promptBox.appendChild(el("div", {}, c.prompt || "(no prompt provided)")); |
| wrap.appendChild(promptBox); |
| |
| if (c.reference_image) { |
| const img = el("img", { |
| class: "ref-img", src: c.reference_image, |
| alt: "reference", |
| onerror: function () { this.style.display = "none"; }, |
| }); |
| wrap.appendChild(img); |
| } |
| return wrap; |
| } |
| |
| function renderVideoGrid(c) { |
| const slots = SHUFFLE[c.id]; |
| const n = slots.length; |
| const grid = el("div", { class: `video-grid cols-${Math.min(n, 4)}` }); |
| slots.forEach((origIdx, slotIdx) => { |
| const v = c.videos[origIdx]; |
| const tile = el("div", { class: "video-tile" }); |
| const labelText = anonymize |
| ? `Video ${String.fromCharCode(65 + slotIdx)}` |
| : `Video ${String.fromCharCode(65 + slotIdx)} · ${v.model}`; |
| tile.appendChild(el("div", { class: "slot-label" }, |
| el("span", { class: "slot-num" }, String.fromCharCode(65 + slotIdx)), |
| el("span", {}, labelText), |
| )); |
| |
| const frame = el("div", { class: "video-frame" }); |
| const video = el("video", { |
| "data-slot": String(slotIdx), |
| "data-case": c.id, |
| preload: "metadata", |
| playsinline: "", |
| controls: "", |
| }); |
| const source = el("source", { src: v.path, type: "video/mp4" }); |
| video.appendChild(source); |
| |
| const empty = el("div", { class: "empty-state", style: "display:none" }, |
| el("div", { class: "icon" }, "▶"), |
| el("div", {}, "no video file at this path"), |
| el("div", { class: "path" }, v.path), |
| el("div", { style: "font-size:11px;color:var(--muted-2);margin-top:4px" }, |
| "drop the .mp4 there, then refresh"), |
| ); |
| frame.appendChild(video); |
| frame.appendChild(empty); |
| |
| |
| video.addEventListener("error", () => { |
| empty.style.display = "flex"; |
| video.style.display = "none"; |
| }); |
| source.addEventListener("error", () => { |
| empty.style.display = "flex"; |
| video.style.display = "none"; |
| }); |
| |
| tile.appendChild(frame); |
| grid.appendChild(tile); |
| }); |
| return grid; |
| } |
| |
| function renderVideoControls(c) { |
| const ctl = el("div", { class: "video-controls" }); |
| ctl.appendChild(el("button", { |
| class: "ctl-btn", onclick: playAll, |
| }, "▶ Play all")); |
| ctl.appendChild(el("button", { |
| class: "ctl-btn", onclick: pauseAll, |
| }, "⏸ Pause")); |
| ctl.appendChild(el("button", { |
| class: "ctl-btn", onclick: restartAll, |
| }, "⏮ Restart")); |
| |
| const loopBtn = el("button", { |
| class: "ctl-btn", id: "loopBtn", |
| onclick: () => toggleLoop(loopBtn), |
| }, "↻ Loop"); |
| ctl.appendChild(loopBtn); |
| |
| const muteBtn = el("button", { |
| class: "ctl-btn", id: "muteBtn", |
| onclick: () => toggleMute(muteBtn), |
| }, "🔇 Mute"); |
| ctl.appendChild(muteBtn); |
| |
| ctl.appendChild(el("span", { style: "margin-left:auto" }, "Speed:")); |
| const speedSel = el("select", { |
| onchange: (e) => setSpeed(parseFloat(e.target.value)), |
| }); |
| ["0.25", "0.5", "0.75", "1", "1.5", "2"].forEach(v => { |
| const opt = el("option", { value: v }, v + "×"); |
| if (v === "1") opt.setAttribute("selected", ""); |
| speedSel.appendChild(opt); |
| }); |
| ctl.appendChild(speedSel); |
| |
| return ctl; |
| } |
| |
| function renderRatingsTable(c) { |
| const slots = SHUFFLE[c.id]; |
| const wrap = el("div", { class: "ratings-table" }); |
| const table = el("table"); |
| const thead = el("thead"); |
| const headRow = el("tr"); |
| headRow.appendChild(el("th", { class: "q-col" }, "Sub-question")); |
| slots.forEach((origIdx, slotIdx) => { |
| const v = c.videos[origIdx]; |
| const label = anonymize |
| ? `Video ${String.fromCharCode(65 + slotIdx)}` |
| : `${String.fromCharCode(65 + slotIdx)} · ${v.model}`; |
| headRow.appendChild(el("th", { class: "video-col" }, label)); |
| }); |
| thead.appendChild(headRow); |
| table.appendChild(thead); |
| |
| const tbody = el("tbody"); |
| for (const dim of DIMENSIONS) { |
| const dimRow = el("tr", { class: "dim-row" }); |
| const dimCell = el("td", { colspan: String(slots.length + 1) }); |
| dimCell.appendChild(el("span", {}, dim.name)); |
| dimCell.appendChild(el("span", { class: "dim-hint" }, dim.hint)); |
| dimRow.appendChild(dimCell); |
| tbody.appendChild(dimRow); |
| |
| for (const q of dim.questions) { |
| const row = el("tr"); |
| const qCell = el("td", { class: "q-cell" }); |
| qCell.appendChild(el("div", { class: "q-name" }, q.name)); |
| qCell.appendChild(el("div", { class: "q-desc" }, q.desc)); |
| row.appendChild(qCell); |
| |
| slots.forEach((origIdx, slotIdx) => { |
| row.appendChild(renderLikertCell(c.id, slotIdx, q.key)); |
| }); |
| tbody.appendChild(row); |
| } |
| } |
| table.appendChild(tbody); |
| wrap.appendChild(table); |
| return wrap; |
| } |
| |
| function renderLikertCell(caseId, slotIdx, qkey) { |
| const cell = el("td", { class: "rate-cell" }); |
| const group = el("div", { class: "likert" }); |
| const cur = getSlotRatings(caseId, slotIdx)[qkey]; |
| for (let s = 1; s <= 5; s++) { |
| const btn = el("button", { |
| "data-score": String(s), |
| title: `${s} — ${SCALE_LABELS[s]}`, |
| onclick: () => { |
| setRating(caseId, slotIdx, qkey, s); |
| |
| Array.from(group.children).forEach((b, i) => { |
| b.classList.toggle("selected", (i + 1) === s); |
| }); |
| }, |
| }, String(s)); |
| if (cur === s) btn.classList.add("selected"); |
| group.appendChild(btn); |
| } |
| cell.appendChild(group); |
| return cell; |
| } |
| |
| |
| |
| |
| function allVideos() { |
| return Array.from(document.querySelectorAll("video[data-slot]")); |
| } |
| function playAll() { allVideos().forEach(v => { try { v.play(); } catch(e){} }); } |
| function pauseAll() { allVideos().forEach(v => v.pause()); } |
| function restartAll() { |
| allVideos().forEach(v => { v.currentTime = 0; try { v.play(); } catch(e){} }); |
| } |
| let LOOPING = false; |
| function toggleLoop(btn) { |
| LOOPING = !LOOPING; |
| allVideos().forEach(v => v.loop = LOOPING); |
| btn.classList.toggle("on", LOOPING); |
| } |
| let MUTED = false; |
| function toggleMute(btn) { |
| MUTED = !MUTED; |
| allVideos().forEach(v => v.muted = MUTED); |
| btn.classList.toggle("on", MUTED); |
| btn.textContent = MUTED ? "🔇 Muted" : "🔊 Sound"; |
| } |
| function setSpeed(rate) { allVideos().forEach(v => v.playbackRate = rate); } |
| function togglePlayAll() { |
| const vids = allVideos(); |
| if (!vids.length) return; |
| const anyPlaying = vids.some(v => !v.paused); |
| if (anyPlaying) pauseAll(); else playAll(); |
| } |
| |
| |
| |
| |
| function goTo(idx) { |
| if (idx < 0 || idx >= CASES.length) return; |
| currentIdx = idx; |
| pauseAll(); |
| renderDetail(); |
| |
| const det = document.getElementById("detail"); |
| if (det) det.scrollTop = 0; |
| } |
| function step(delta) { goTo((currentIdx + delta + CASES.length) % CASES.length); } |
| |
| function updateProgress() { |
| const { done, expected } = totalProgress(); |
| const pct = expected > 0 ? (done / expected) * 100 : 0; |
| const fill = document.getElementById("progressFill"); |
| const txt = document.getElementById("progressText"); |
| if (fill) fill.style.width = pct.toFixed(1) + "%"; |
| if (txt) txt.innerHTML = |
| `<span class="num">${done}</span> / ${expected} ratings (${pct.toFixed(0)}%)`; |
| } |
| |
| function updateFooterSummary() { |
| const c = CASES[currentIdx]; |
| if (!c) return; |
| const p = caseProgress(c); |
| const el = document.getElementById("footerSummary"); |
| if (!el) return; |
| el.innerHTML = |
| `Case <span class="num">${currentIdx + 1}</span> / ${CASES.length} · ` + |
| `<span class="num">${p.done}</span> / ${p.expected} ratings on this case` + |
| (p.done === p.expected && p.expected > 0 ? " · ✓ complete" : ""); |
| } |
| |
| function updateCaseStatus(caseId) { |
| const c = CASES.find(x => x.id === caseId); |
| if (!c) return; |
| const p = caseProgress(c); |
| const row = document.querySelector(`[data-case-id="${caseId}"]`); |
| if (!row) return; |
| row.classList.remove("partial", "complete"); |
| if (p.done === p.expected && p.expected > 0) row.classList.add("complete"); |
| else if (p.done > 0) row.classList.add("partial"); |
| const counter = row.querySelector("span:last-child"); |
| if (counter) counter.textContent = `${p.done}/${p.expected}`; |
| updateFooterSummary(); |
| } |
| |
| |
| |
| |
| function toggleAnon() { |
| anonymize = !anonymize; |
| |
| SHUFFLE = {}; |
| buildShuffle(CASES); |
| saveState(); |
| renderShell(); |
| } |
| |
| function resetCurrentCase() { |
| const c = CASES[currentIdx]; |
| if (!confirm(`Reset all ratings for ${c.id}?`)) return; |
| delete RATINGS[c.id]; |
| saveState(); |
| renderDetail(); |
| document.querySelector(`[data-case-id="${c.id}"]`)?.classList.remove("partial", "complete"); |
| updateProgress(); |
| updateCaseStatus(c.id); |
| showToast("Cleared ratings for " + c.id); |
| } |
| |
| function resetAll() { |
| if (!confirm("Reset all ratings, all cases, and rater id? This cannot be undone.")) return; |
| RATINGS = {}; |
| SHUFFLE = {}; |
| RATER = { id: genRaterId(), note: "", started_at: new Date().toISOString() }; |
| buildShuffle(CASES); |
| saveState(); |
| renderShell(); |
| showToast("All ratings reset"); |
| } |
| |
| function exportJSON() { |
| |
| const out = { |
| rater: RATER, |
| anonymize_used: anonymize, |
| exported_at: new Date().toISOString(), |
| rubric: DIMENSIONS, |
| cases: CASES.map(c => { |
| const slots = SHUFFLE[c.id] || c.videos.map((_, i) => i); |
| return { |
| id: c.id, |
| prompt: c.prompt, |
| videos: slots.map((origIdx, slotIdx) => ({ |
| slot: String.fromCharCode(65 + slotIdx), |
| slot_index: slotIdx, |
| original_index: origIdx, |
| model: c.videos[origIdx].model, |
| path: c.videos[origIdx].path, |
| ratings: (RATINGS[c.id] || {})[slotIdx] || {}, |
| })), |
| }; |
| }), |
| }; |
| const blob = new Blob([JSON.stringify(out, null, 2)], { type: "application/json" }); |
| const url = URL.createObjectURL(blob); |
| const a = el("a", { |
| href: url, |
| download: `user_study_${RATER.id || "anon"}_${new Date().toISOString().slice(0, 10)}.json`, |
| }); |
| document.body.appendChild(a); a.click(); a.remove(); |
| URL.revokeObjectURL(url); |
| showToast("Exported JSON"); |
| } |
| |
| |
| |
| |
| function renderEmptyApp(reason) { |
| document.getElementById("app").innerHTML = ` |
| <div class="empty-app"> |
| <h2>Video user study</h2> |
| <p style="color:var(--bad)">${escapeHtml(reason)}</p> |
| <p>This page expects a manifest at <code>./cases.json</code> describing the cases to rate. |
| A working example lives at <code>./cases.example.json</code>.</p> |
| <h3 style="color:var(--accent-strong);font-size:13px;text-transform:uppercase;letter-spacing:0.4px">cases.json format</h3> |
| <pre>[ |
| { |
| "id": "case_001", |
| "prompt": "A cat in a chef's hat tossing pancakes...", |
| "reference_image": "refs/case_001.png", // optional |
| "videos": [ |
| { "model": "ours", "path": "videos/case_001_ours.mp4" }, |
| { "model": "baseline_a", "path": "videos/case_001_a.mp4" } |
| ] |
| } |
| ]</pre> |
| <p>Then serve this directory and open the page:</p> |
| <pre>cd evals/user_study |
| python -m http.server 8765 |
| # open http://localhost:8765/</pre> |
| <p style="color:var(--muted-2)">Tip: missing video files render as a striped "no video" placeholder, so you |
| can scaffold the rating UI before any real <code>.mp4</code>s exist.</p> |
| </div>`; |
| } |
| |
| |
| |
| |
| document.addEventListener("keydown", e => { |
| if (e.target.tagName === "INPUT" || e.target.tagName === "TEXTAREA") return; |
| if (e.metaKey || e.ctrlKey || e.altKey) return; |
| switch (e.key) { |
| case "ArrowLeft": case "k": e.preventDefault(); step(-1); break; |
| case "ArrowRight": case "j": e.preventDefault(); step(1); break; |
| case " ": e.preventDefault(); togglePlayAll(); break; |
| case "r": case "R": e.preventDefault(); restartAll(); break; |
| case "l": case "L": |
| e.preventDefault(); |
| const lb = document.getElementById("loopBtn"); |
| if (lb) toggleLoop(lb); |
| break; |
| } |
| }); |
| |
| |
| |
| |
| async function fetchManifest() { |
| for (const path of ["./cases.json", "./cases.example.json"]) { |
| try { |
| const resp = await fetch(path, { cache: "no-cache" }); |
| if (resp.ok) { |
| const data = await resp.json(); |
| return { data, path }; |
| } |
| } catch (e) { } |
| } |
| return null; |
| } |
| |
| (async function init() { |
| const found = await fetchManifest(); |
| if (!found) { |
| renderEmptyApp("Could not load ./cases.json or ./cases.example.json — " + |
| "either run via python -m http.server, or drop a manifest in this directory."); |
| return; |
| } |
| CASES = found.data || []; |
| if (!Array.isArray(CASES) || CASES.length === 0) { |
| renderEmptyApp("Manifest is empty — add at least one case to cases.json."); |
| return; |
| } |
| |
| |
| const prior = loadState(); |
| if (prior) { |
| RATER = prior.rater || { id: genRaterId(), note: "", started_at: new Date().toISOString() }; |
| anonymize = prior.anonymize !== false; |
| RATINGS = prior.ratings || {}; |
| SHUFFLE = prior.shuffle || {}; |
| } else { |
| RATER = { id: genRaterId(), note: "", started_at: new Date().toISOString() }; |
| } |
| |
| buildShuffle(CASES); |
| saveState(); |
| renderShell(); |
| if (found.path.endsWith("example.json")) { |
| showToast("Loaded cases.example.json — drop a real cases.json to override"); |
| } |
| |
| |
| probeSyncEndpoint(); |
| })(); |
| </script> |
| </body> |
| </html> |
|
|
