sandbox-bench

1#!/usr/bin/env node2// Post-hoc (re-)analysis of runs at the boot level. Reads ONLY the3// results db: a run-dir argument is imported (idempotently) and4// verified first, a bare run id analyzes what the db already holds.5//   node bench-analyze.mjs <runDir|runId> [--base <arm>] [--cand <arm>]6import fs from 'node:fs'7import path from 'node:path'8import { analyzeE2eRows } from './bench-stats.mjs'9import { openDb, importRun, loadRows, runMeta, verify } from './bench-db.mjs'10 11const argv = process.argv.slice(2)12const target = argv.find((a) => !a.startsWith('--'))13const get = (name) => {14  const i = argv.indexOf(name)15  return i >= 0 ? argv[i + 1] : undefined16}17if (!target) {18  console.error(19    'usage: node bench-analyze.mjs <runDir|runId> [--base arm] [--cand arm]'20  )21  process.exit(1)22}23const db = openDb()24let runId25if (fs.existsSync(target)) {26  runId = importRun(db, target).runId27} else {28  runId = path.basename(target)29}30const { failures, notes } = verify(db, runId)31for (const n of notes) console.log(`note: ${n}`)32if (failures.length) {33  console.error(`RESULTS DB VERIFY FAILED:\n  ${failures.join('\n  ')}`)34  process.exit(1)35}36const rows = loadRows(db, runId)37const arms = [...new Set(rows.map((r) => r.arm))]38if (arms.length !== 2) {39  console.error(`expected exactly 2 arms, found: ${arms.join(', ')}`)40  process.exit(1)41}42// Which arm is base? Row order cannot answer this (ABBA runs the43// candidate first), so require an explicit source: the run's meta,44// a conventional name, or the --base flag.45const meta = runMeta(db, runId)?.meta46const base =47  get('--base') ??48  (meta && arms.includes(meta.base) ? meta.base : undefined) ??49  ['base', 'synced'].find((n) => arms.includes(n))50if (!base) {51  console.error(52    `cannot determine the base arm from {${arms.join(', ')}}: ` +53      'no meta, no arm named "base"/"synced" — pass --base <arm>'54  )55  process.exit(1)56}57const cand = get('--cand') ?? arms.find((a) => a !== base)58const boots = new Set(rows.map((r) => r.vm)).size59console.log(`${runId}  boots=${boots}  arms: ${base} (base) vs ${cand}`)60if (meta?.pr)61  console.log(62    `PR: ${meta.pr.title ? `"${meta.pr.title}" — ` : ''}${meta.pr.url}`63  )64for (const a of meta?.arms ?? [])65  if (a.title) console.log(`  ${a.name}: "${a.title}"`)66if (rows[0].route === '') {67  // Micro (sandbox-ab) samples: phase holds the payload name.68  analyzeE2eRows(rows, base, cand, ['mean', 'p50', 'p95', 'p99', 'gcMs'])69} else if (meta?.suite === 'ssr') {70  analyzeE2eRows(rows, base, cand, [71    'rps',72    'mean',73    'median',74    'p95',75    'p99',76    'gcMs',77    'heapMb',78  ])79} else {80  // No p99 for e2e: the load phases are far too small for it.81  analyzeE2eRows(rows, base, cand, [82    'rps',83    'median',84    'mean',85    'p95',86    'ttfb',87    'docKb',88    'gzipKb',89    'flightKb',90    'rss',91    'rssHw',92  ])93}94