app/src/content/embeds/failure-features.html

<!-- Failure Features: Dumbbell chart with CV + Holdout AUROC and top features -->
<div class="failure-features"></div>
<style>
  .failure-features { position: relative; width: 100%; min-height: 420px; }
  .failure-features svg { display: block; width: 100%; }
  .failure-features .tick text { fill: var(--text-color); font-size: 11px; }
  .failure-features .tick line, .failure-features .domain { stroke: var(--border-color); }
  .failure-features .grid line { stroke: var(--border-color); opacity: 0.15; }
  .failure-features .grid .domain { display: none; }
  .failure-features .axis-label { fill: var(--text-color); font-size: 13px; font-weight: 500; }
  .failure-features .tooltip {
    position: absolute; top: 0; left: 0; pointer-events: none; padding: 10px 14px; border-radius: 8px;
    font-size: 12px; line-height: 1.6; border: 1px solid var(--border-color);
    background: var(--surface-bg); color: var(--text-color);
    box-shadow: 0 4px 20px rgba(0,0,0,0.12), 0 0 0 1px rgba(0,0,0,0.04);
    backdrop-filter: blur(12px); -webkit-backdrop-filter: blur(12px);
    opacity: 0; transition: opacity 0.15s ease;
    z-index: 100; max-width: 260px;
    font-variant-numeric: tabular-nums;
  }
  .failure-features .ref-line { stroke-dasharray: 6,4; }
  .failure-features .feature-pill {
    font-family: 'SF Mono', 'Fira Code', 'Consolas', monospace;
    font-size: 10px;
  }
  .failure-features .legend-item text { fill: var(--text-color); font-size: 11px; }
  @media (max-width: 600px) {
    .failure-features { min-height: 340px; }
    .failure-features .tooltip { max-width: 200px; }
    .failure-features .feature-pill { display: none; }
  }
</style>
<script>
(() => {
  const ensureD3 = (cb) => {
    if (window.d3 && typeof window.d3.select === 'function') return cb();
    let s = document.getElementById('d3-cdn-script');
    if (!s) { s = document.createElement('script'); s.id = 'd3-cdn-script'; s.src = 'https://cdn.jsdelivr.net/npm/d3@7/dist/d3.min.js'; document.head.appendChild(s); }
    s.addEventListener('load', () => cb(), { once: true });
  };

  const bootstrap = () => {
    const container = document.querySelector('.failure-features:not([data-mounted])');
    if (!container) return;
    container.dataset.mounted = 'true';
    const d3 = window.d3;

    const data = [
      { name: 'tau2 telecom', states: 43, cvAuroc: 0.946, holdout: 0.941, topFeature: 'message length before human handoff', features: 129 },
      { name: 'WebArena', states: 25, cvAuroc: 0.882, holdout: 0.903, topFeature: 'whether the run ends on a click', features: 50 },
      { name: 'ATBench', states: 15, cvAuroc: 0.864, holdout: 0.894, topFeature: 'whether the run ends on an assistant turn', features: 36 },
      { name: 'AgentNet', states: 25, cvAuroc: 0.886, holdout: 0.890, topFeature: 'a write-then-click step', features: 73 },
      { name: 'tau2 airline', states: 18, cvAuroc: 0.826, holdout: 0.864, topFeature: 'a cancel-reservation step', features: 70 },
      { name: 'SWE-agent', states: 25, cvAuroc: 0.806, holdout: 0.799, topFeature: 'edit message length', features: 39 },
      { name: 'tau2 retail', states: 19, cvAuroc: 0.752, holdout: 0.779, topFeature: 'longest user message', features: 52 },
      { name: 'OSWorld', states: 27, cvAuroc: 0.779, holdout: 0.774, topFeature: 'trace length', features: 79 },
      { name: 'SWE-smith', states: 10, cvAuroc: 0.636, holdout: 0.703, topFeature: 'longest tool message', features: 54 },
    ].sort((a, b) => b.holdout - a.holdout);

    const ours = '#3d5a80';
    const slate = '#8fa6c4';

    const margin = { top: 38, right: 150, bottom: 48, left: 110 };
    const tip = document.createElement('div');
    tip.className = 'tooltip';
    container.appendChild(tip);

    let hasAnimated = false;

    function render() {
      container.querySelectorAll('svg').forEach(s => s.remove());
      const rect = container.getBoundingClientRect();
      const W = Math.max(400, Math.round(rect.width));
      const barH = 34;
      const H = margin.top + margin.bottom + data.length * (barH + 5);
      const w = Math.max(0, W - margin.left - margin.right);
      const h = Math.max(0, H - margin.top - margin.bottom);

      const svg = d3.select(container).insert('svg', '.tooltip')
        .attr('width', W).attr('height', Math.max(0, H));
      const g = svg.append('g').attr('transform', `translate(${margin.left},${margin.top})`);

      const x = d3.scaleLinear().domain([0.5, 1.0]).range([0, w]);
      const y = d3.scaleBand().domain(data.map(d => d.name)).range([0, h]).padding(0.3);

      // Grid
      g.append('g').attr('class', 'grid')
        .attr('transform', `translate(0,${h})`)
        .call(d3.axisBottom(x).ticks(5).tickSize(Math.max(0, -h)).tickFormat(''));

      // Random baseline (0.5) dashed line
      g.append('line').attr('class', 'ref-line')
        .attr('x1', x(0.5)).attr('x2', x(0.5)).attr('y1', -8).attr('y2', h)
        .attr('stroke', 'var(--text-color)').attr('stroke-width', 1).attr('stroke-opacity', 0.3);
      g.append('text')
        .attr('x', x(0.5)).attr('y', -12)
        .attr('fill', 'var(--text-color)').attr('font-size', '10px')
        .attr('text-anchor', 'middle').attr('opacity', 0.45).text('random (0.5)');

      // Bottom axis
      g.append('g').attr('transform', `translate(0,${h})`).call(d3.axisBottom(x).ticks(5));
      // Left axis
      g.append('g').call(d3.axisLeft(y).tickSize(0)).select('.domain').remove();

      g.append('text').attr('class', 'axis-label')
        .attr('x', w / 2).attr('y', h + 40).attr('text-anchor', 'middle')
        .text('AUROC');

      // --- Horizontal legend at top ---
      const leg = g.append('g').attr('transform', `translate(0, -26)`);
      // Holdout
      leg.append('circle').attr('cx', 0).attr('cy', 0).attr('r', 5).attr('fill', ours);
      leg.append('text').attr('class', 'legend-item').attr('x', 10).attr('y', 4)
        .attr('fill', 'var(--text-color)').attr('font-size', '11px').text('Holdout AUROC');
      // CV
      leg.append('circle').attr('cx', 112).attr('cy', 0).attr('r', 5)
        .attr('fill', 'none').attr('stroke', slate).attr('stroke-width', 2);
      leg.append('text').attr('class', 'legend-item').attr('x', 122).attr('y', 4)
        .attr('fill', 'var(--text-color)').attr('font-size', '11px').text('5-fold CV AUROC');
      // Connector
      leg.append('line').attr('x1', 230).attr('x2', 254).attr('y1', 0).attr('y2', 0)
        .attr('stroke', slate).attr('stroke-width', 2).attr('stroke-opacity', 0.4);
      leg.append('text').attr('class', 'legend-item').attr('x', 260).attr('y', 4)
        .attr('fill', 'var(--text-color)').attr('font-size', '11px').attr('opacity', 0.55).text('gap');

      const animate = !hasAnimated;
      const dur = 500;

      // --- Dumbbell rows ---
      data.forEach((d, idx) => {
        const cy = y(d.name) + y.bandwidth() / 2;
        const delay = idx * 60;
        const xCV = x(d.cvAuroc);
        const xHO = x(d.holdout);
        const x0 = x(0.5);

        // Connecting line (CV to Holdout)
        const line = g.append('line')
          .attr('y1', cy).attr('y2', cy)
          .attr('stroke', slate).attr('stroke-width', 2).attr('stroke-opacity', 0.35)
          .attr('stroke-linecap', 'round');
        if (animate) {
          line.attr('x1', x0).attr('x2', x0)
            .transition().duration(dur).delay(delay).ease(d3.easeCubicOut)
            .attr('x1', Math.min(xCV, xHO)).attr('x2', Math.max(xCV, xHO));
        } else {
          line.attr('x1', Math.min(xCV, xHO)).attr('x2', Math.max(xCV, xHO));
        }

        // CV AUROC (open circle)
        const cvCircle = g.append('circle')
          .attr('cx', animate ? x0 : xCV).attr('cy', cy)
          .attr('fill', 'var(--page-bg)').attr('stroke', slate).attr('stroke-width', 2)
          .attr('cursor', 'pointer');
        if (animate) {
          cvCircle.attr('r', 0)
            .transition().duration(dur).delay(delay).ease(d3.easeCubicOut)
            .attr('cx', xCV).attr('r', 5);
        } else {
          cvCircle.attr('r', 5);
        }

        // Holdout AUROC (filled circle)
        const hoCircle = g.append('circle')
          .attr('cx', animate ? x0 : xHO).attr('cy', cy)
          .attr('fill', ours).attr('stroke', 'var(--page-bg)').attr('stroke-width', 2)
          .attr('cursor', 'pointer');
        if (animate) {
          hoCircle.attr('r', 0)
            .transition().duration(dur).delay(delay + 80).ease(d3.easeCubicOut)
            .attr('cx', xHO).attr('r', 6);
        } else {
          hoCircle.attr('r', 6);
        }

        // Hover target (invisible wide rect)
        const hitArea = g.append('rect')
          .attr('x', 0).attr('y', cy - y.bandwidth() / 2)
          .attr('width', Math.max(0, w)).attr('height', Math.max(0, y.bandwidth()))
          .attr('fill', 'transparent').attr('cursor', 'pointer');

        hitArea
          .on('mouseenter', function(ev) {
            cvCircle.attr('r', 7); hoCircle.attr('r', 8);
            line.attr('stroke-opacity', 0.6).attr('stroke-width', 3);
            tip.innerHTML = `<strong>${d.name}</strong><br/>States: ${d.states} | Features: ${d.features}<br/>CV AUROC: ${d.cvAuroc.toFixed(3)}<br/>Holdout AUROC: <strong>${d.holdout.toFixed(3)}</strong><br/>Top feature: <code style="font-size:11px;background:color-mix(in srgb, var(--text-color) 8%, transparent);padding:2px 5px;border-radius:4px">${d.topFeature}</code>`;
            tip.style.opacity = '1';
          })
          .on('mousemove', function(ev) {
            const [mx, my] = d3.pointer(ev, container);
            const flipX = mx > rect.width * 0.6;
            tip.style.transform = `translate(${flipX ? mx - 200 : mx + 14}px, ${my - 14}px)`;
          })
          .on('mouseleave', function() {
            cvCircle.attr('r', 5); hoCircle.attr('r', 6);
            line.attr('stroke-opacity', 0.35).attr('stroke-width', 2);
            tip.style.opacity = '0';
          });

        // Feature pill on right
        const pillG = g.append('g')
          .attr('transform', `translate(${w + 10}, ${cy})`);
        const pillText = pillG.append('text')
          .attr('class', 'feature-pill')
          .attr('x', 6).attr('y', 4)
          .attr('fill', 'var(--text-color)').attr('opacity', 0.65)
          .text(d.topFeature);

        // Compute pill background after text renders
        setTimeout(() => {
          const bbox = pillText.node().getBBox();
          pillG.insert('rect', 'text')
            .attr('x', bbox.x - 5).attr('y', bbox.y - 2)
            .attr('width', Math.max(0, bbox.width + 10)).attr('height', Math.max(0, bbox.height + 4))
            .attr('rx', 4).attr('fill', 'var(--text-color)').attr('opacity', 0.05);
        }, 0);
      });

      hasAnimated = true;
    }

    render();
    if (window.ResizeObserver) {
      let resizeTimer;
      new ResizeObserver(() => {
        clearTimeout(resizeTimer);
        resizeTimer = setTimeout(render, 80);
      }).observe(container);
    }
  };

  if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', () => ensureD3(bootstrap), { once: true });
  else ensureD3(bootstrap);
})();
</script>