ossrules.md

sandbox-bench

Benchmark React or Next.js changes on Vercel Sandbox VMs with paired A/B statistics: react PR/commit vs base, or Next.js PR/commit vs base, measured end-to-end through the bench/render-pipeline app (rps, latency, p95; TTFB, RSS and document/Flight bytes when the Next side captures them) and, for React changes, through the react repo's flight-ssr-bench fixture (Node AND Edge web-streams paths, Fizz and Flight+Fizz). Use whenever the user asks to bench, perf test, or A/B a React PR, a react-server-dom / Flight / vendored React change, or a Next.js PR ("is this PR faster", "does this regress RSC?", "measure the perf impact of <commit>"), even if they don't say "benchmark" — any request to quantify a server-side performance difference between two revisions belongs here. Runs remotely (laptop-free), applies correctness gates before measuring, and reports boot-level confidence intervals.

Install
npx skills add 'https://github.com/vercel/next.js/tree/canary/.agents/skills/sandbox-bench'
Download bundle ↓
canary · bfcf687 ↗Scanned 2026-09-15

Contributors

GitHub-linked commit authors for this SKILL.md at the saved revision. Co-authors and history before file renames are not included.

File history ↗

scripts/sandbox-gate.mjs

scripts/sandbox-gate.mjsBrowse 13 files
View on GitHub
3,670 tokens

12,704 bytes
Token encoding: o200k_base
Snapshot bfcf687

← Back to SKILL.md
#!/usr/bin/env node// Correctness gate for React arms before spending bench compute: each// arm gets a sandbox VM booted from the shared react build-env snapshot// (created lazily if missing), extracts the arm's tree over the// snapshot's node_modules, and runs the jest suite in prod mode (the// channel the bench measures).// A bench number from an arm that fails its tests is meaningless —// always gate hand-assembled or conflict-resolved arms.//// Usage: node sandbox-gate.mjs --arms name=ref[,name=ref...]//        [--pattern <jest pattern>, default: full suite] [--extra "yarn flow dom-node"]//        [--snapshot snap_x] [--dry-run]import { execFile, spawn } from 'node:child_process'import { promisify } from 'node:util'import fs from 'node:fs'import os from 'node:os'import path from 'node:path'import crypto from 'node:crypto'import { loadConfig, sandboxScope, ensureReactRepo } from './config.mjs' const execFileP = promisify(execFile)const DRY_RUN = process.argv.includes('--dry-run')const CONFIG = loadConfig({ requireScope: !DRY_RUN })const REACT = ensureReactRepo(CONFIG)const VERCEL = CONFIG.vercelBinconst SCOPE = CONFIG.team ? sandboxScope(CONFIG) : []const SETUP_VERSION = 'v1-al2023-node24-jdk21'const REACT_SNAP_CACHE = path.join(CONFIG.cacheDir, 'react-snap') function parseArgs() {  const args = { arms: [], snapshot: undefined, extra: [], pattern: '' }  const argv = process.argv.slice(2)  for (let i = 0; i < argv.length; i++) {    if (argv[i] === '--arms') {      for (const spec of argv[++i].split(',')) {        const [name, ref] = spec.split('=')        if (!name || !ref) throw new Error(`bad arm spec: ${spec}`)        args.arms.push({ name, ref })      }    } else if (argv[i] === '--snapshot') {      args.snapshot = argv[++i]    } else if (argv[i] === '--extra') {      // Extra gate command(s) run in the repo root after the jest gates,      // e.g. --extra "yarn flow dom-node".      args.extra.push(argv[++i])    } else if (argv[i] === '--pattern') {      args.pattern = argv[++i]    } else if (argv[i] === '--dry-run') {      // handled globally    } else {      throw new Error(`unknown arg: ${argv[i]}`)    }  }  if (args.arms.length === 0) throw new Error('need --arms name=ref[,...]')  if (args.pattern !== '' && !/^[A-Za-z0-9|_.-]+$/.test(args.pattern)) {    // The pattern is interpolated into the remote command line; reject    // anything that could smuggle extra jest flags (--passWithNoTests    // would turn an empty selection into a PASS).    throw new Error(      `--pattern must match [A-Za-z0-9|_.-]+, got: ${args.pattern}`    )  }  return args} async function sb(args, opts = {}) {  const scoped = ['sandbox', ...args]  const sep = scoped.indexOf('--')  scoped.splice(sep < 0 ? scoped.length : sep, 0, ...SCOPE)  const { stdout, stderr } = await execFileP(VERCEL, scoped, {    maxBuffer: 64 * 1024 * 1024,    ...opts,  })  // The CLI prints some results (e.g. snapshot ids) on stderr; callers  // that parse exact values (size checks) must keep stdout-only.  return opts.withStderr ? `${stdout}\n${stderr}` : stdout} async function git(args) {  const { stdout } = await execFileP('git', ['-C', REACT, ...args], {    maxBuffer: 256 * 1024 * 1024,  })  return stdout.trim()} async function rmVm(name) {  try {    await sb(['rm', name])  } catch (e) {    process.stderr.write(`warning: could not remove ${name}: ${e.message}\n`)  }} // Same detached-nohup-and-poll shape as sandbox-e2e.mjs runDetached:// exec streams drop flakily on multi-minute silences, so never hold one.async function runDetached(vm, tag, script, deadlineMin) {  const local = path.join(os.tmpdir(), `gate-${vm}.sh`)  fs.writeFileSync(    local,    // EXIT trap, not ERR: see sandbox-e2e.mjs runDetached.    `trap 'code=$?; if [ $code -eq 0 ]; then echo LOOPOK; else echo LOOPFAIL; fi > /vercel/sandbox/loop.done' EXIT\nset -e\n${script}\n`  )  await sb(['cp', local, `${vm}:/vercel/sandbox/loop.sh`])  fs.rmSync(local, { force: true })  await sb([    'exec',    vm,    '--timeout',    '2m',    '--',    'bash',    '-c',    'rm -f /vercel/sandbox/loop.done /vercel/sandbox/loop.log; nohup bash /vercel/sandbox/loop.sh >/vercel/sandbox/loop.log 2>&1 & echo kicked',  ])  let offset = 0  let failures = 0  const deadline = Date.now() + deadlineMin * 60_000  while (true) {    await new Promise((r) => setTimeout(r, 45_000))    if (Date.now() > deadline) throw new Error(`${tag}: deadline exceeded`)    let out    try {      out = await sb([        'exec',        vm,        '--timeout',        '2m',        '--',        'bash',        '-c',        `tail -c +${offset + 1} /vercel/sandbox/loop.log | head -c 200000; printf '\\n@@SIZE %s @@DONE %s\\n' "$(stat -c %s /vercel/sandbox/loop.log 2>/dev/null || echo 0)" "$(cat /vercel/sandbox/loop.done 2>/dev/null || echo no)"`,      ])      failures = 0    } catch (e) {      if (++failures >= 6)        throw new Error(          `${tag}: ${failures} consecutive poll failures: ${e.message.slice(0, 200)}`        )      continue    }    const m = out.match(/@@SIZE (\d+) @@DONE (\S+)/)    const body = out.slice(0, out.lastIndexOf('\n@@SIZE'))    for (const l of body.split('\n')) {      if (l) process.stderr.write(`[${tag}] ${l}\n`)    }    if (m) {      offset = Math.min(Number(m[1]), offset + 200000)      if (m[2] === 'LOOPOK') return      if (m[2] === 'LOOPFAIL')        throw new Error(          `${tag}: remote gate failed; tail:\n${body.slice(-3000)}`        )    }  }} async function gateArm(arm, snapshot, tmp, extra = [], pattern = '') {  const sha = await git(['rev-parse', arm.ref])  const short = sha.slice(0, 12)  const vm = `sbench-gate-${arm.name}-${Date.now().toString(36)}`  process.stderr.write(`[${arm.name}] vm ${vm} gating ${short}\n`)  try {    await sb([      'create',      '--name',      vm,      '--snapshot',      snapshot,      '--vcpus',      '16',      '--timeout',      '2h',      '--non-persistent',      '--network-policy',      'allow-all',      '--tag',      'purpose=sandbox-bench',      '--silent',    ])  } catch (e) {    if (!/vcpu/i.test(e.message)) throw e    await sb([      'create',      '--name',      vm,      '--snapshot',      snapshot,      '--vcpus',      '8',      '--timeout',      '2h',      '--non-persistent',      '--network-policy',      'allow-all',      '--tag',      'purpose=sandbox-bench',      '--silent',    ])  }  try {    const src = path.join(tmp, `gate-src-${arm.name}.tgz`)    await execFileP('bash', [      '-c',      `git -C ${REACT} archive ${sha} | gzip -1 > ${src}`,    ])    await sb(['cp', src, `${vm}:/vercel/sandbox/src.tgz`])    const srcSize = fs.statSync(src).size    fs.rmSync(src, { force: true })    // sandbox cp can succeed on missing remote files: size-check remotely.    const check = await sb([      'exec',      vm,      '--timeout',      '2m',      '--',      'bash',      '-c',      `stat -c %s /vercel/sandbox/src.tgz`,    ])    if (Number(check.trim()) !== srcSize) {      throw new Error(        `${arm.name}: uploaded src.tgz size ${check.trim()} != local ${srcSize}`      )    }    await runDetached(      vm,      `gate:${arm.name}`,      // pipefail: every gate command is piped through tail, which would      // otherwise mask its exit status from set -e (debugged 2026-07-19:      // a failing flow run still printed GATE PASS).      `set -o pipefail\n` +        `cd /vercel/sandbox/react\n` +        `find . -mindepth 1 -maxdepth 1 ! -name node_modules -exec rm -rf {} +\n` +        `tar -xzf ../src.tgz && rm -f ../src.tgz\n` +        `echo "== ${arm.name} ${short} prod gate (${pattern || 'full suite'}) =="\n` +        `yarn test --prod ${pattern} --ci 2>&1 | tail -40\n` +        extra          .map(            (cmd) =>              `echo "== ${arm.name} extra: ${cmd} =="\n${cmd} 2>&1 | tail -30\n`          )          .join('') +        `echo "== ${arm.name} GATE PASS =="\n`,      90    )    return { arm: arm.name, sha: short, pass: true }  } finally {    await rmVm(vm)  }} // Build-env snapshot: node_modules installed + JDK for the closure// compiler, keyed on yarn.lock. Created once per lockfile, reused by// every later gate.const snapshotPromises = new Map() async function ensureReactBuildSnapshot(refSha) {  // Raw bytes, exactly as the e2e harness hashes it: a trim here would  // silently fork the cache key and build every snapshot twice.  const lock = (    await execFileP('git', ['-C', REACT, 'show', `${refSha}:yarn.lock`], {      maxBuffer: 256 * 1024 * 1024,    })  ).stdout  const memoKey = crypto.createHash('sha256').update(lock).digest('hex')  if (!snapshotPromises.has(memoKey)) {    snapshotPromises.set(memoKey, ensureReactBuildSnapshotForLock(refSha, lock))  }  return snapshotPromises.get(memoKey)} async function ensureReactBuildSnapshotForLock(refSha, lock) {  const key = crypto    .createHash('sha256')    .update(SETUP_VERSION + lock)    .digest('hex')    .slice(0, 16)  const file = path.join(REACT_SNAP_CACHE, `snap-${key}`)  if (fs.existsSync(file)) {    const id = fs.readFileSync(file, 'utf8').trim()    try {      await sb(['snapshots', 'get', id])      return id    } catch {}  }  const vm = `sbench-snapbuild-${Date.now().toString(36)}`  console.error(    'creating react build-env snapshot (one-time for this yarn.lock)...'  )  await sb([    'create',    '--name',    vm,    '--runtime',    'node24',    '--vcpus',    '8',    '--timeout',    '45m',    '--non-persistent',    '--network-policy',    'allow-all',    '--tag',    'purpose=sandbox-bench',    '--silent',  ])  try {    const src = path.join(os.tmpdir(), `react-snap-src-${key}.tgz`)    await execFileP('bash', [      '-c',      `git -C ${REACT} archive ${refSha} | gzip -1 > ${src}`,    ])    await sb(['cp', src, `${vm}:/vercel/sandbox/src.tgz`])    fs.rmSync(src, { force: true })    await sb([      'exec',      vm,      '--timeout',      '10m',      '--sudo',      '--',      'dnf',      'install',      '-y',      '-q',      'java-21-amazon-corretto-headless',    ])    await runDetached(      vm,      'react-snap',      `mkdir -p /vercel/sandbox/react && cd /vercel/sandbox/react && tar -xzf ../src.tgz && rm -f ../src.tgz\n` +        `npm i -g yarn >/dev/null 2>&1\n` +        `yarn install --frozen-lockfile --ignore-engines >/dev/null 2>&1\n` +        `echo react env ready\n`,      30    )    const out = await sb(['snapshot', vm, '--stop', '--expiration', '30d'], {      withStderr: true,    })    const snapId = out.match(/snap_[A-Za-z0-9]+/)?.[0]    if (!snapId)      throw new Error(`could not parse snapshot id from: ${out.slice(-500)}`)    fs.mkdirSync(REACT_SNAP_CACHE, { recursive: true })    fs.writeFileSync(file, `${snapId}\n`)    return snapId  } finally {    await rmVm(vm)  }} async function main() {  const args = parseArgs()  if (DRY_RUN) {    console.log(      `[dry-run] gate plan: ${args.arms.map((a) => `${a.name}=${a.ref}`).join(', ')}`    )    console.log(      `[dry-run] per arm: 1 VM from react build-env snapshot (lazy-created), ` +        `yarn test --prod ${args.pattern} --ci` +        (args.extra.length ? `, extra: ${args.extra.join(' && ')}` : '')    )    console.log('[dry-run] PASS requires: verified test counts (pipefail on).')    process.exit(0)  }  // Each arm gates against ITS OWN lockfile's environment: arms with  // different yarn.locks must not share node_modules.  const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'sandbox-gate-'))  try {    const results = await Promise.allSettled(      args.arms.map(async (arm) => {        const sha = await git(['rev-parse', arm.ref])        const snapshot = args.snapshot ?? (await ensureReactBuildSnapshot(sha))        return gateArm(arm, snapshot, tmp, args.extra, args.pattern)      })    )    let failed = 0    for (let i = 0; i < results.length; i++) {      const r = results[i]      if (r.status === 'fulfilled') {        console.log(`GATE ${args.arms[i].name}: PASS (${r.value.sha})`)        console.log(          `  bench this exact sha: --arms cand=${r.value.sha} (a ref can move between gate and bench)`        )      } else {        failed++        console.log(          `GATE ${args.arms[i].name}: FAIL — ${r.reason.message.slice(0, 2500)}`        )      }    }    process.exit(failed ? 1 : 0)  } finally {    fs.rmSync(tmp, { recursive: true, force: true })  }} main().catch((e) => {  console.error(e)  process.exit(1)})
Referenced from SKILL.md
SKILL.mdView in source ↗
Source excerpt starting at line 87.
```shnode scripts/sandbox-gate.mjs --arms cand=<ref>```
← All Next.js skillsProject instructions →