{"$schema":"https://ui.shadcn.com/schema/registry-item.json","name":"eval-results","type":"registry:ui","title":"Eval Results","description":"Pass rates per test case across two runs, with the regressions surfaced first and a sample size honest enough to say when a delta means nothing.","author":"Scrim UI (https://scrimui.dev)","categories":["feedback"],"docs":"https://scrimui.dev/components/eval-results","dependencies":[],"registryDependencies":[],"files":[{"path":"src/showcase/eval-results/eval-results.tsx","content":"\"use client\";\n\nimport * as React from \"react\";\n\n/**\n * Pass rates for a suite of test cases, this run against the last one.\n *\n * The table is easy. What makes an eval view useful or useless is what it\n * does with the two numbers per row:\n *\n * **A delta is not a result.** 8/10 to 9/10 is one sample changing its mind,\n * and on a non-deterministic system that happens for free. So every row\n * carries its sample count, and any change smaller than the noise floor is\n * rendered as *unchanged* rather than as a green arrow. A green arrow that\n * means nothing is worse than no arrow, because someone will ship on it.\n *\n * **Regressions first, always.** Sorting alphabetically, or by pass rate,\n * buries the two rows that are the reason anyone opened the page. Improvements\n * are pleasant; regressions are the job.\n *\n * **A new case is not an improvement.** A row with no baseline has nothing to\n * compare against, and rendering it as +100% is how a suite that grew looks\n * like a model that got better.\n *\n * **Partial results are partial.** While a run is in flight the summary is\n * computed over the cases that have finished, and cases finish in whatever\n * order they were scheduled — which is not random with respect to difficulty\n * if anything is batched. So the header says how many have reported, and the\n * unfinished rows stay visible rather than being filtered out.\n */\n\nexport type EvalCase = {\n  id: string;\n  name: string;\n  /** 0–1. Undefined means this case is new: nothing to compare against. */\n  baseline?: number;\n  /** 0–1. Undefined means it has not reported yet in this run. */\n  current?: number;\n  /** How many times the case was run. A delta over five samples is a rumour. */\n  samples: number;\n};\n\nexport type EvalResultsProps = {\n  cases: EvalCase[];\n  baselineLabel?: string;\n  currentLabel?: string;\n  /**\n   * Deltas smaller than this are shown as unchanged. Default 0.05 — with the\n   * sample counts most suites actually run, anything tighter is noise wearing\n   * a colour.\n   */\n  noiseFloor?: number;\n  /** Below this, no delta is called at all regardless of size. */\n  minSamples?: number;\n  running?: boolean;\n  className?: string;\n};\n\n/* ------------------------------------------------------------------ */\n/* Icons                                                               */\n/* ------------------------------------------------------------------ */\n\nfunction ArrowIcon({ up, ...props }: { up: boolean } & React.SVGProps<SVGSVGElement>) {\n  return (\n    <svg viewBox=\"0 0 24 24\" fill=\"none\" stroke=\"currentColor\" strokeWidth=\"2.5\" strokeLinecap=\"round\" strokeLinejoin=\"round\" width=\"11\" height=\"11\" {...props}>\n      {up ? <path d=\"M12 19V5M5 12l7-7 7 7\" /> : <path d=\"M12 5v14M5 12l7 7 7-7\" />}\n    </svg>\n  );\n}\n\n/* ------------------------------------------------------------------ */\n/* Classification                                                      */\n/* ------------------------------------------------------------------ */\n\ntype Verdict = \"regression\" | \"improvement\" | \"unchanged\" | \"new\" | \"pending\" | \"undercounted\";\n\nfunction verdictOf(c: EvalCase, noiseFloor: number, minSamples: number): Verdict {\n  if (c.current === undefined) return \"pending\";\n  if (c.baseline === undefined) return \"new\";\n  if (c.samples < minSamples) return \"undercounted\";\n  const delta = c.current - c.baseline;\n  if (Math.abs(delta) < noiseFloor) return \"unchanged\";\n  return delta < 0 ? \"regression\" : \"improvement\";\n}\n\n/* Regressions, then anything still uncertain, then improvements, then the\n   rows nobody needs to look at. Within a group, worst delta first. */\nconst ORDER: Record<Verdict, number> = {\n  regression: 0,\n  undercounted: 1,\n  pending: 2,\n  new: 3,\n  improvement: 4,\n  unchanged: 5,\n};\n\nfunction pct(v: number): string {\n  return `${Math.round(v * 100)}%`;\n}\n\n/* ------------------------------------------------------------------ */\n/* EvalResults                                                         */\n/* ------------------------------------------------------------------ */\n\nexport function EvalResults({\n  cases,\n  baselineLabel = \"baseline\",\n  currentLabel = \"this run\",\n  noiseFloor = 0.05,\n  minSamples = 10,\n  running = false,\n  className = \"\",\n}: EvalResultsProps) {\n  const rows = React.useMemo(() => {\n    return cases\n      .map((c) => ({ c, verdict: verdictOf(c, noiseFloor, minSamples) }))\n      .sort((x, y) => {\n        const byGroup = ORDER[x.verdict] - ORDER[y.verdict];\n        if (byGroup !== 0) return byGroup;\n        const dx = (x.c.current ?? 0) - (x.c.baseline ?? 0);\n        const dy = (y.c.current ?? 0) - (y.c.baseline ?? 0);\n        return dx - dy;\n      });\n  }, [cases, noiseFloor, minSamples]);\n\n  const reported = cases.filter((c) => c.current !== undefined);\n  /* Averaged over reported cases only, and labelled as such. A mean that\n     silently treats an unfinished case as zero is a number that improves as\n     the run progresses for reasons that have nothing to do with the model. */\n  const mean =\n    reported.length === 0\n      ? undefined\n      : reported.reduce((sum, c) => sum + (c.current ?? 0), 0) / reported.length;\n  const regressions = rows.filter((r) => r.verdict === \"regression\").length;\n\n  return (\n    <div className={`overflow-hidden rounded-xl border border-zinc-200 bg-white dark:border-zinc-800 dark:bg-zinc-900 ${className}`}>\n      <div className=\"flex flex-wrap items-baseline gap-x-3 gap-y-1 border-b border-zinc-100 px-3.5 py-2.5 dark:border-zinc-800\">\n        <span className=\"text-sm font-medium text-zinc-900 dark:text-zinc-100\">\n          {mean === undefined ? \"—\" : pct(mean)}\n        </span>\n        <span className=\"text-[11px] text-zinc-500 dark:text-zinc-400\">\n          mean over {reported.length} of {cases.length} cases\n        </span>\n        {running && (\n          <span className=\"inline-flex items-center gap-1.5 text-[11px] text-blue-600 dark:text-blue-400\">\n            <span className=\"h-2 w-2 animate-pulse rounded-full bg-current\" />\n            running\n          </span>\n        )}\n        {regressions > 0 && (\n          <span className=\"ml-auto text-[11px] font-medium text-red-600 dark:text-red-400\">\n            {regressions} regression{regressions === 1 ? \"\" : \"s\"}\n          </span>\n        )}\n      </div>\n\n      <div className=\"overflow-x-auto\">\n        <table className=\"w-full text-left text-[13px]\">\n          <thead>\n            <tr className=\"text-[11px] uppercase tracking-wide text-zinc-400 dark:text-zinc-500\">\n              <th className=\"px-3.5 py-2 font-medium\">Case</th>\n              <th className=\"px-2 py-2 text-right font-medium\">{baselineLabel}</th>\n              <th className=\"px-2 py-2 text-right font-medium\">{currentLabel}</th>\n              <th className=\"px-3.5 py-2 text-right font-medium\">Δ</th>\n            </tr>\n          </thead>\n          <tbody>\n            {rows.map(({ c, verdict }) => (\n              <tr\n                key={c.id}\n                className=\"border-t border-zinc-100 align-top dark:border-zinc-800/80\"\n              >\n                <td className=\"px-3.5 py-2\">\n                  <span className=\"text-zinc-800 dark:text-zinc-100\">{c.name}</span>\n                  <span className=\"ml-2 text-[11px] tabular-nums text-zinc-400 dark:text-zinc-500\">\n                    n={c.samples}\n                  </span>\n                </td>\n                <td className=\"px-2 py-2 text-right tabular-nums text-zinc-500 dark:text-zinc-400\">\n                  {c.baseline === undefined ? \"—\" : pct(c.baseline)}\n                </td>\n                <td className=\"px-2 py-2 text-right tabular-nums text-zinc-800 dark:text-zinc-100\">\n                  {c.current === undefined ? \"—\" : pct(c.current)}\n                </td>\n                <td className=\"px-3.5 py-2 text-right\">\n                  <Delta c={c} verdict={verdict} minSamples={minSamples} />\n                </td>\n              </tr>\n            ))}\n          </tbody>\n        </table>\n      </div>\n\n      <p className=\"border-t border-zinc-100 px-3.5 py-2 text-[11px] leading-4 text-zinc-400 dark:border-zinc-800 dark:text-zinc-500\">\n        Changes under {pct(noiseFloor)} are shown as unchanged, and no delta is called below{\" \"}\n        {minSamples} samples. Both are guesses about your noise, not facts about it — measure the\n        variance of a repeated run and set them from that.\n      </p>\n    </div>\n  );\n}\n\nfunction Delta({ c, verdict, minSamples }: { c: EvalCase; verdict: Verdict; minSamples: number }) {\n  if (verdict === \"pending\") {\n    return <span className=\"text-[11px] text-zinc-400 dark:text-zinc-500\">waiting</span>;\n  }\n  if (verdict === \"new\") {\n    /* Not +100%. A suite that grew is not a model that improved. */\n    return <span className=\"text-[11px] text-zinc-400 dark:text-zinc-500\">new case</span>;\n  }\n  if (verdict === \"undercounted\") {\n    return (\n      <span className=\"text-[11px] text-amber-600 dark:text-amber-500\" title={`Fewer than ${minSamples} samples.`}>\n        too few to call\n      </span>\n    );\n  }\n  if (verdict === \"unchanged\") {\n    return <span className=\"text-[11px] text-zinc-400 dark:text-zinc-500\">unchanged</span>;\n  }\n\n  const delta = (c.current ?? 0) - (c.baseline ?? 0);\n  const up = delta > 0;\n  return (\n    <span\n      className={`inline-flex items-center justify-end gap-1 text-[11px] font-medium tabular-nums ${\n        up ? \"text-emerald-700 dark:text-emerald-400\" : \"text-red-600 dark:text-red-400\"\n      }`}\n    >\n      <ArrowIcon up={up} />\n      {up ? \"+\" : \"−\"}\n      {Math.abs(Math.round(delta * 100))}%\n    </span>\n  );\n}\n","type":"registry:ui","target":"components/ui/eval-results.tsx"}]}