/* Workload matrix, cell detail, source composition, task explorer. */ (function () { 'use strict'; var U = window.U, D = U.D, h = U.h; var FORM_LABEL = { 'single-step': 'Single-step', 'multi-step': 'Multi-step', 'multi-step+tools': 'Multi-step + tools' }; var MOD_LABEL = { text: 'Text', multimodal: 'Multimodal' }; var selected = null, cellBtns = {}, exampleIdx = {}; function forms() { var f = []; U.GROUPS.forEach(function (g) { if (f.indexOf(g.form) < 0) f.push(g.form); }); return f; } function mods() { var m = []; U.GROUPS.forEach(function (g) { if (m.indexOf(g.modality) < 0) m.push(g.modality); }); return m; } function groupAt(mod, form) { return U.GROUPS.filter(function (g) { return g.modality === mod && g.form === form; })[0]; } function mixbar(scoring, lg) { var bar = h('span', { class: 'mixbar' + (lg ? ' mixbar--lg' : ''), attrs: { 'aria-hidden': 'true' } }); U.SCORING_ORDER.forEach(function (k) { if (!scoring[k]) return; bar.appendChild(h('span', { style: { background: U.scoringVar(k), flex: scoring[k] + ' 1 0' } })); }); return bar; } function mixText(scoring) { return U.SCORING_ORDER.filter(function (k) { return scoring[k]; }).map(function (k) { return scoring[k] + ' ' + U.scoringLabel[k].toLowerCase(); }).join(', '); } function scoringLegend() { var lg = h('div', { class: 'legend' }, h('span', { class: 'legend__title', text: 'Scoring' })); U.SCORING_ORDER.forEach(function (k) { lg.appendChild(h('span', { class: 'legend__item' }, h('span', { class: 'legend__sw', style: { background: U.scoringVar(k) } }), U.scoringLabel[k])); }); return lg; } /* ---------------- matrix ---------------- */ function buildMatrix() { var host = U.$('#matrix'); var grid = h('div', { class: 'matrix__grid' }); grid.appendChild(h('span', { class: 'matrix__corner', text: 'input ↓ / execution →' })); forms().forEach(function (f) { grid.appendChild(h('span', { class: 'matrix__col', text: FORM_LABEL[f] || f })); }); grid.appendChild(h('span', { class: 'matrix__axis', attrs: { 'aria-hidden': 'true' } }, h('span', { text: 'increasing execution complexity' }))); mods().forEach(function (mod) { grid.appendChild(h('span', { class: 'matrix__row', text: MOD_LABEL[mod] || mod })); forms().forEach(function (f) { var g = groupAt(mod, f); var srcNames = g.sources.map(function (k) { return U.source(k).name; }); /* the accessible name is the visible text plus a hidden gloss, so it contains the visible label (WCAG 2.5.3) */ var btn = h('button', { class: 'wcell', style: { '--c': U.wlVar(g.key) }, attrs: { type: 'button', 'aria-pressed': 'false', 'aria-controls': 'cell-detail' } }, h('span', { class: 'wcell__count', text: String(g.count) }), h('span', { class: 'wcell__unit', text: 'tasks' }), h('span', { class: 'wcell__label', text: g.label }), h('span', { class: 'wcell__sr', text: ' (' + g.longLabel + ')' }), h('span', { class: 'wcell__src', text: srcNames.join(', ') }), h('span', { class: 'wcell__sr', text: ' Scoring: ' + mixText(g.scoring) + '.' }), mixbar(g.scoring)); btn.addEventListener('click', function () { select(g.key, true); }); U.hoverTip(btn, function () { return { title: g.longLabel, rows: [{ label: 'Tasks', value: String(g.count), color: U.css('--wl-' + g.key) }].concat(U.SCORING_ORDER.filter(function (k) { return g.scoring[k]; }).map(function (k) { return { label: U.scoringLabel[k] + ' scoring', value: String(g.scoring[k]), color: U.css('--score-' + k) }; })), note: srcNames.join(', ') }; }); cellBtns[g.key] = btn; grid.appendChild(btn); }); }); host.appendChild(grid); host.appendChild(h('div', { class: 'matrix__foot' }, scoringLegend(), h('span', { class: 'matrix__hint', text: 'select a cell to dive in' }))); } function select(key, focusDetail) { selected = key; Object.keys(cellBtns).forEach(function (k) { cellBtns[k].setAttribute('aria-pressed', String(k === key)); }); renderDetail(key); } function renderDetail(key) { var g = U.group(key), box = U.$('#cell-detail'); box.textContent = ''; box.appendChild(h('div', { class: 'cd__head' }, h('span', { class: 'cd__sw', style: { background: U.wlVar(key) } }), h('h3', { class: 'cd__title', text: g.longLabel }), h('span', { class: 'cd__count', text: g.count + ' tasks · ' + g.applicableModels + ' models' }))); var grid = h('div', { class: 'cd__grid' }); /* sources + scoring */ var left = h('div', { class: 'cd__block' }, h('h4', { text: 'Sources' })); var srcs = h('div', { class: 'cd__srcs' }); g.sources.forEach(function (k) { var s = U.source(k); srcs.appendChild(h('span', { class: 'pill', attrs: { title: s.capability } }, s.name, h('span', { class: 'chip__n', text: String(s.count) }))); }); left.appendChild(srcs); left.appendChild(h('h4', { text: 'Scoring modes', style: { 'margin-top': '14px' } })); var mixRows = h('div', { class: 'cd__mixrows' }); U.SCORING_ORDER.forEach(function (k) { if (!g.scoring[k]) return; mixRows.appendChild(h('span', null, h('span', { class: 'legend__sw', style: { background: U.scoringVar(k) } }), g.scoring[k] + ' ' + U.scoringLabel[k].toLowerCase())); }); left.appendChild(h('div', { class: 'cd__mix' }, mixbar(g.scoring, true), mixRows)); grid.appendChild(left); /* top models (Table 1) */ var right = h('div', { class: 'cd__block' }, h('h4', { text: 'Top models (audited accuracy, %)' })); var models = D.models.filter(function (m) { return m.acc[key] != null; }); var ranks = U.rank(models, function (m) { return m.acc[key]; }); var top = models.filter(function (m) { return ranks.get(m) <= 3; }).sort(function (a, b) { return b.acc[key] - a.acc[key]; }); var max = 100; var ol = h('ol', { class: 'top3' }); var items = []; top.forEach(function (m) { var r = ranks.get(m); var li = h('li', null, h('span', { class: 'medal medal--' + r + ' top3__rank' }, h('span', { class: 'sr-only', text: 'Rank ' }), String(r)), h('span', { class: 'top3__name' }, m.name, h('span', { class: 'top3__vendor', text: m.vendor + ' · ' + (m.deployment === 'api' ? 'API' : 'Local') })), h('span', { class: 'top3__val', text: U.pct(m.acc[key]) }), h('span', { class: 'top3__bar', attrs: { 'aria-hidden': 'true' }, style: { width: (m.acc[key] / max * 100) + '%', background: U.wlVar(key) } })); U.hoverTip(li, function () { return { title: m.name, rows: [{ label: g.longLabel + ' accuracy', value: U.pct(m.acc[key]) + '%', color: U.css('--wl-' + key) }], note: m.vendor + ' · ' + (m.deployment === 'api' ? 'API' : 'Local') + ' · rank ' + r + ' of ' + models.length + ' models (Table 1, audited accuracy)' }; }); items.push(li); ol.appendChild(li); }); /* one Tab stop for the mini chart; arrow keys move between its bars */ if (items.length) U.roving(items); right.appendChild(ol); var locals = models.filter(function (m) { return m.deployment === 'local'; }).sort(function (a, b) { return b.acc[key] - a.acc[key]; }); if (locals.length) right.appendChild(h('p', { class: 'cd__local' }, 'Best local model: ', h('b', { text: locals[0].shortName }), ' (' + U.pct(locals[0].acc[key]) + ')')); grid.appendChild(right); box.appendChild(grid); /* example task */ var exs = D.examples.filter(function (e) { return e.group === key; }); if (exs.length) { var idx = exampleIdx[key] || 0; var ex = h('div', { class: 'example' }); var tabs = h('div', { class: 'example__tabs' }, h('h4', { text: 'Example task from the repo' })); var seg = h('div', { class: 'seg', attrs: { role: 'radiogroup', 'aria-label': 'Example task' } }); tabs.appendChild(seg); ex.appendChild(tabs); var body = h('div'); ex.appendChild(body); U.seg(seg, exs.map(function (e, i) { return { value: i, label: e.sourceName }; }), idx, function (v) { exampleIdx[key] = v; drawExample(body, exs[v]); }); drawExample(body, exs[idx]); box.appendChild(ex); } } function drawExample(body, e) { body.textContent = ''; var card = h('div', { class: 'example__card example__card--taped' + (e.imageFile ? ' has-img' : '') }); if (e.imageFile) { var zb = U.zoomable(U.IMG + e.imageFile, e.imageAlt || '', e.imageAlt || '', 'task image for ' + e.id); card.appendChild(h('div', { class: 'example__img' }, zb)); } var txt = h('div'); txt.appendChild(h('p', { class: 'example__prompt', text: e.promptExcerpt, attrs: { lang: /[一-鿿]/.test(e.promptExcerpt) ? 'zh' : null } })); if (e.englishGloss) txt.appendChild(h('p', { class: 'example__gloss', text: 'English wording in the paper: “' + e.englishGloss + '”' })); /* a translation made for the page (the paper does not give one): labelled as such */ else if (e.englishTranslation) txt.appendChild(h('p', { class: 'example__gloss', text: 'In English (our translation): “' + e.englishTranslation + '”' })); var meta = h('div', { class: 'example__meta' }, h('span', { class: 'pill pill--soft example__id', text: e.id }), h('span', { class: 'pill', text: e.scoring + ' scoring' }), h('span', { class: 'pill', text: e.timeoutSeconds + ' s budget' }), h('span', { class: 'pill', text: 'artifact: ' + e.artifact })); txt.appendChild(meta); var notes = [e.excerptNote]; if (e.paperRef) notes.push(e.paperRef + '.'); txt.appendChild(h('p', { class: 'example__note', text: notes.join(' ') })); card.appendChild(txt); body.appendChild(card); } /* ---------------- sources ---------------- */ var chipBtns = {}, segEls = {}, pinned = null; function buildSources() { U.$('#sources-sub').textContent = D.stats.sources + ' public benchmarks contribute ' + D.stats.tasks + ' tasks. Bar width is the number of tasks each source contributes; color shows its workload group.'; var lg = U.$('#sources-legend'); U.GROUPS.forEach(function (g) { lg.appendChild(h('span', { class: 'legend__item' }, h('span', { class: 'legend__sw', style: { background: U.wlVar(g.key) } }), g.label)); }); var bar = U.$('#compbar'); D.sources.forEach(function (s) { var g = U.group(s.group); /* pointer-only view of the composition: the source chips below are the keyboard and screen-reader equivalent (same tooltip, same pin action), so the bar is aria-hidden */ var seg = h('div', { class: 'compbar__seg', style: { background: U.wlVar(s.group), flex: s.count + ' 1 0', 'min-width': '0' } }); seg.addEventListener('pointermove', function (e) { U.tip.show(srcTip(s), e.clientX, e.clientY, seg); }); seg.addEventListener('pointerleave', function () { U.tip.hide(seg); }); seg.addEventListener('pointerenter', function () { hot(s.key, true); }); seg.addEventListener('pointerleave', function () { hot(s.key, false); }); seg.addEventListener('click', function () { pin(s.key); }); segEls[s.key] = seg; bar.appendChild(seg); }); var wall = U.$('#chipwall'); U.GROUPS.forEach(function (g) { /* lane title breaks only between "modality · form", never inside a part; the count never splits */ var title = h('span', { class: 'lane__title' }); g.longLabel.split(' · ').forEach(function (part, i, arr) { title.appendChild(h('span', { class: 'nowrap', text: part + (i < arr.length - 1 ? ' ·' : '') })); if (i < arr.length - 1) title.appendChild(document.createTextNode(' ')); }); var lane = h('div', { class: 'lane' }, h('div', { class: 'lane__head' }, h('span', { class: 'legend__sw', style: { background: U.wlVar(g.key) } }), title, h('small', { text: g.count + ' tasks' }))); var chips = h('div', { class: 'lane__chips' }); g.sources.forEach(function (k) { var s = U.source(k); var c = h('button', { class: 'chip', attrs: { type: 'button', 'aria-pressed': 'false', 'aria-controls': 'source-info', 'aria-label': s.name + ', ' + s.count + ' tasks' } }, s.name, h('span', { class: 'chip__n', text: String(s.count) })); U.hoverTip(c, function () { return srcTip(s); }); c.addEventListener('pointerenter', function () { hot(k, true); }); c.addEventListener('pointerleave', function () { hot(k, false); }); c.addEventListener('click', function () { pin(k); }); chipBtns[k] = c; chips.appendChild(c); }); lane.appendChild(chips); wall.appendChild(lane); }); U.$('#source-info').appendChild(h('p', { class: 'source-info__empty', text: 'select a benchmark to read what it tests' })); /* a named, focusable scroller with the caption above it, like the other table views */ U.tableView(U.$('#sources-table'), U.table([ { label: 'Benchmark', get: function (s) { return s.name; } }, { label: 'Capability', get: function (s) { return s.capability; } }, { label: '# Tasks', num: true, get: function (s) { return String(s.count); } }, { label: 'Group', get: function (s) { return U.group(s.group).label; } }, { label: 'Scoring', get: function (s) { return s.scoring; } } ], D.sources, 'Source benchmarks (Table 9)')); } function srcTip(s) { var g = U.group(s.group); return { title: s.name, rows: [ { label: 'Tasks', value: String(s.count), color: U.css('--wl-' + s.group) }, { label: 'Group', value: g.label }, { label: 'Scoring', value: s.scoring } ], note: s.capability }; } function hot(k, on) { if (chipBtns[k]) chipBtns[k].classList.toggle('is-hot', on); if (segEls[k]) segEls[k].classList.toggle('is-hot', on); } function pin(k) { pinned = pinned === k ? null : k; Object.keys(chipBtns).forEach(function (x) { chipBtns[x].setAttribute('aria-pressed', String(x === pinned)); }); var box = U.$('#source-info'); box.textContent = ''; if (!pinned) { box.appendChild(h('p', { class: 'source-info__empty', text: 'select a benchmark to read what it tests' })); return; } var s = U.source(pinned), g = U.group(s.group); var cite = s.citation || {}; box.appendChild(h('div', { class: 'source-info__card' }, h('h4', { text: s.fullName && s.fullName !== s.name ? s.name + ' — ' + s.fullName : s.name }), h('div', { class: 'source-info__meta' }, h('span', { class: 'pill' }, h('span', { class: 'dot', style: { background: U.wlVar(s.group) } }), g.label), h('span', { class: 'pill', text: s.count + ' tasks' }), h('span', { class: 'pill', text: s.scoring }), h('span', { class: 'pill pill--soft', text: s.capability })), h('p', { text: s.description }), cite.title ? h('p', { class: 'source-info__cite', text: (cite.text ? cite.text + '. ' : '') + cite.title + (cite.venue ? '. ' + cite.venue : '') }) : null)); } /* ---------------- explorer ---------------- */ function buildExplorer() { U.$('#explorer-sub').textContent = D.tasks.length + ' tasks · id and name, source, group, scoring, level and time budget'; var fg = U.$('#f-group'), fs = U.$('#f-source'), fsc = U.$('#f-scoring'), fq = U.$('#f-q'); fg.appendChild(h('option', { text: 'All groups', attrs: { value: '' } })); U.GROUPS.forEach(function (g) { fg.appendChild(h('option', { text: g.label, attrs: { value: g.key } })); }); fs.appendChild(h('option', { text: 'All sources', attrs: { value: '' } })); D.sources.forEach(function (s) { fs.appendChild(h('option', { text: s.name, attrs: { value: s.key } })); }); fsc.appendChild(h('option', { text: 'All modes', attrs: { value: '' } })); ['Auto', 'Hybrid', 'Judge'].forEach(function (s) { fsc.appendChild(h('option', { text: s, attrs: { value: s } })); }); var table = U.$('#task-table'); table.appendChild(h('caption', { class: 'sr-only', text: 'Task metadata from the repository (no prompts or answers)' })); var thead = h('thead', null, h('tr', null, h('th', { text: 'Task id · name', attrs: { scope: 'col' } }), h('th', { text: 'Source', attrs: { scope: 'col' } }), h('th', { text: 'Group', attrs: { scope: 'col' } }), h('th', { text: 'Scoring', attrs: { scope: 'col' } }), h('th', { text: 'Level', attrs: { scope: 'col' } }), h('th', { class: 'num', text: 'Time budget (s)', attrs: { scope: 'col' } }))); var tb = h('tbody'); table.appendChild(thead); table.appendChild(tb); function draw() { var q = fq.value.trim().toLowerCase(); var rows = D.tasks.filter(function (t) { return (!fg.value || t.group === fg.value) && (!fs.value || t.source === fs.value) && (!fsc.value || t.scoring === fsc.value) && (!q || t.id.toLowerCase().indexOf(q) >= 0 || (t.name || '').toLowerCase().indexOf(q) >= 0); }); tb.textContent = ''; var frag = document.createDocumentFragment(); rows.forEach(function (t) { var g = U.group(t.group); frag.appendChild(h('tr', null, h('th', { class: 'mono', attrs: { scope: 'row' } }, t.id, t.name ? h('span', { class: 'task-name', text: t.name }) : null), h('td', { text: U.source(t.source).name }), h('td', null, h('span', { class: 'legend__item' }, h('span', { class: 'legend__sw', style: { background: U.wlVar(t.group), width: '10px', height: '10px' } }), g.label)), h('td', { text: t.scoring }), h('td', { text: t.level || '—' }), h('td', { class: 'num', text: String(t.timeoutSeconds) }))); }); tb.appendChild(frag); U.$('#explorer-count').textContent = 'Showing ' + rows.length + ' of ' + D.tasks.length + ' tasks'; } [fg, fs, fsc].forEach(function (el) { el.addEventListener('change', draw); }); fq.addEventListener('input', draw); /* the 233-row table is only built the first time the explorer is opened */ var ex = U.$('#explorer'), built = false; function ensure() { if (!built && ex.open) { built = true; draw(); } } ex.addEventListener('toggle', ensure); ensure(); } U.initWorkloads = function () { buildMatrix(); select(D.groupOrder[0]); buildSources(); buildExplorer(); }; })();