sandbox-bench

1#!/usr/bin/env node2// Recover a run whose local launcher died (session teardown, crash):3// the remote VMs keep executing their detached loops regardless, so the4// data is usually still there. Reads the run's status.json for the VM5// names, waits for each VM's loop to finish, downloads its results,6// removes the VM, and runs the boot-level analysis.7//8//   node bench-collect.mjs <runDir> [--deadline-min 280]9import { execFile } from 'node:child_process'10import { promisify } from 'node:util'11import fs from 'node:fs'12import path from 'node:path'13import { loadConfig, sandboxScope } from './config.mjs'14 15const execFileP = promisify(execFile)16const CONFIG = loadConfig()17const SCOPE = sandboxScope(CONFIG)18const argv = process.argv.slice(2)19const dir = argv.find((a) => !a.startsWith('--'))20const deadlineMin = argv.includes('--deadline-min')21  ? Number(argv[argv.indexOf('--deadline-min') + 1])22  : 28023if (!dir || !fs.existsSync(path.join(dir, 'status.json'))) {24  console.error('usage: node bench-collect.mjs <runDir with status.json>')25  process.exit(1)26}27const statusPath = path.join(dir, 'status.json')28const st = JSON.parse(fs.readFileSync(statusPath, 'utf8'))29const vms = Object.keys(st.vms ?? {})30function writeStatus(patch) {31  Object.assign(st, patch, { updatedAt: new Date().toISOString() })32  fs.writeFileSync(statusPath, JSON.stringify(st, null, 2))33}34writeStatus({ phase: 'collecting (recovery)', pid: process.pid })35if (vms.length === 0) {36  console.error(37    'status.json lists no VMs — the run died before measurement; nothing to collect. ' +38      'Check for a leaked snapshot builder with sandbox-sweep.mjs and relaunch the run.'39  )40  process.exit(1)41}42 43async function sb(args) {44  const scoped = ['sandbox', ...args]45  const sep = scoped.indexOf('--')46  scoped.splice(sep < 0 ? scoped.length : sep, 0, ...SCOPE)47  const { stdout } = await execFileP(CONFIG.vercelBin, scoped, {48    maxBuffer: 1 << 26,49  })50  return stdout51}52 53const pending = new Map(vms.map((vm, i) => [vm, i]))54const collected = []55const deadline = Date.now() + deadlineMin * 6000056while (pending.size > 0 && Date.now() < deadline) {57  for (const [vm, idx] of [...pending]) {58    let out59    try {60      out = await sb([61        'exec',62        vm,63        '--timeout',64        '2m',65        '--',66        'bash',67        '-c',68        'cat /vercel/sandbox/loop.done 2>/dev/null || echo RUNNING',69      ])70    } catch (e) {71      // VM gone (timed out or removed): its data is lost; say so and move on.72      console.error(73        `${vm}: unreachable (${e.message.split('\n')[0].slice(0, 100)}) — boot lost`74      )75      st.vms[vm] = { ...st.vms[vm], state: 'lost' }76      writeStatus({})77      pending.delete(vm)78      continue79    }80    const state = out.trim().split('\n').pop()81    if (state === 'RUNNING') continue82    if (state === 'LOOPOK') {83      const local = path.join(dir, `results-vm${idx}.jsonl`)84      await sb(['cp', `${vm}:/vercel/sandbox/results.jsonl`, local])85      if (fs.existsSync(local) && fs.statSync(local).size > 500) {86        collected.push(local)87        st.vms[vm] = { ...st.vms[vm], state: 'done' }88        writeStatus({})89        console.error(`${vm}: collected -> ${local}`)90      } else {91        console.error(`${vm}: results too small; keeping VM for inspection`)92        pending.delete(vm)93        continue94      }95    } else {96      const tail = await sb([97        'exec',98        vm,99        '--timeout',100        '2m',101        '--',102        'bash',103        '-c',104        'tail -c 1500 /vercel/sandbox/loop.log',105      ])106      console.error(`${vm}: loop ended ${state}; log tail:\n${tail}`)107    }108    try {109      await sb(['rm', vm])110      console.error(`${vm}: removed`)111    } catch (e) {112      console.error(113        `${vm}: rm failed: ${e.message.split('\n')[0].slice(0, 100)}`114      )115    }116    pending.delete(vm)117  }118  if (pending.size > 0) await new Promise((r) => setTimeout(r, 60000))119}120for (const [vm] of pending)121  console.error(`${vm}: deadline exceeded, left running`)122 123if (collected.length === 0) {124  console.error('nothing collected')125  process.exit(1)126}127writeStatus({128  phase: `done (recovered ${collected.length}/${vms.length} boots)`,129})130console.log(`\ncollected ${collected.length}/${vms.length} boots; analysis:`)131const { stdout } = await execFileP(132  'node',133  [134    path.join(135      path.dirname(new URL(import.meta.url).pathname),136      'bench-analyze.mjs'137    ),138    dir,139  ],140  { maxBuffer: 1 << 24 }141)142console.log(stdout)143