runtime-behavior-probe

1import { spawnSync } from 'node:child_process';2import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';3import { dirname, join, resolve } from 'node:path';4import { createRequire } from 'node:module';5import { pathToFileURL } from 'node:url';6 7/**8 * Disposable probe workflow for openai-agents-js:9 * 1. Put this file in a temporary directory created with `mktemp -d`.10 * 2. Add a sibling `tsconfig.json` that extends `${process.cwd()}/tsconfig.examples.json`11 *    and includes only `./probe.ts`.12 * 3. From the repository root, run `pnpm exec tsc --noEmit -p /tmp/.../tsconfig.json`.13 * 4. If that passes, run `pnpm exec tsx /tmp/.../probe.ts`.14 * 5. Only switch to `dist/` imports and `pnpm build` when packaged output is the thing under test.15 */16 17type ProbeMode = 'single-shot' | 'repeat-N' | 'warm-up + repeat-N';18type ResultFlag = 'unexpected' | 'negative' | 'expected' | 'blocked';19 20type CaseResult = {21  case_id: string;22  mode: ProbeMode;23  is_warmup: boolean;24  observation_summary: string;25  result_flag: ResultFlag;26  metrics: Record<string, unknown>;27  error: string | null;28  total_latency_s?: number;29  first_token_latency_s?: number;30};31 32const SCENARIO = 'replace-me';33const RUN_LABEL = 'replace-me';34const MODE: ProbeMode = 'single-shot';35const APPROVED_ENV_VARS: string[] = [];36const OUTPUT_DIR_ENV = 'PROBE_OUTPUT_DIR';37 38const RESULTS: CaseResult[] = [];39const repoRequire = createRequire(join(process.cwd(), 'package.json'));40 41function gitValue(...args: string[]): string {42  const result = spawnSync('git', args, {43    cwd: process.cwd(),44    encoding: 'utf8',45  });46  if (result.status !== 0) {47    return 'unknown';48  }49  const value = result.stdout.trim();50  return value || 'unknown';51}52 53function outputDir(): string | null {54  const value = process.env[OUTPUT_DIR_ENV];55  return value ? resolve(value) : null;56}57 58function writeJson(path: string, payload: unknown): void {59  mkdirSync(dirname(path), { recursive: true });60  writeFileSync(path, `${JSON.stringify(payload, null, 2)}\n`);61}62 63function emit(kind: string, payload: Record<string, unknown> = {}): void {64  console.log(65    JSON.stringify({66      ts: Number((Date.now() / 1000).toFixed(3)),67      kind,68      ...payload,69    }),70  );71}72 73function findNearestPackageJson(startPath: string): string | null {74  let current = resolve(startPath);75  while (true) {76    const candidate = join(current, 'package.json');77    if (existsSync(candidate)) {78      return candidate;79    }80    const parent = resolve(current, '..');81    if (parent === current) {82      return null;83    }84    current = parent;85  }86}87 88function resolvePackageVersion(packageName: string): string | null {89  try {90    const packageJsonPath = repoRequire.resolve(`${packageName}/package.json`);91    return JSON.parse(readFileSync(packageJsonPath, 'utf8')).version ?? null;92  } catch {93    try {94      const entryPath = repoRequire.resolve(packageName);95      const packageJsonPath = findNearestPackageJson(resolve(entryPath, '..'));96      if (!packageJsonPath) {97        return null;98      }99      return JSON.parse(readFileSync(packageJsonPath, 'utf8')).version ?? null;100    } catch {101      return null;102    }103  }104}105 106export async function importRepoModule(107  repoRelativePath: string,108): Promise<Record<string, unknown>> {109  const absolutePath = resolve(process.cwd(), repoRelativePath);110  return import(pathToFileURL(absolutePath).href);111}112 113function runtimeContext(): Record<string, unknown> {114  const approvedEnvVars = Object.fromEntries(115    APPROVED_ENV_VARS.map((name) => [116      name,117      process.env[name] ? 'set' : 'unset',118    ]),119  );120 121  const packageVersions = Object.fromEntries(122    ['openai', '@openai/agents', '@openai/agents-core']123      .map((name) => [name, resolvePackageVersion(name)])124      .filter((entry): entry is [string, string] => Boolean(entry[1])),125  );126 127  return {128    scenario: SCENARIO,129    run_label: RUN_LABEL,130    mode: MODE,131    cwd: process.cwd(),132    script_path: resolve(process.argv[1] ?? ''),133    node_executable: process.execPath,134    node_version: process.version,135    platform: process.platform,136    git_commit: gitValue('rev-parse', 'HEAD'),137    git_branch: gitValue('rev-parse', '--abbrev-ref', 'HEAD'),138    pnpm_execpath: process.env.npm_execpath ?? null,139    package_versions: packageVersions,140    approved_env_vars: approvedEnvVars,141    output_dir: outputDir(),142  };143}144 145function startCase(146  caseId: string,147  options: { mode?: ProbeMode; note?: string } = {},148): void {149  emit('case_start', {150    case_id: caseId,151    mode: options.mode ?? MODE,152    note: options.note ?? null,153  });154}155 156function recordCaseResult(157  caseId: string,158  observationSummary: string,159  resultFlag: ResultFlag,160  options: {161    mode?: ProbeMode;162    isWarmup?: boolean;163    totalLatencyS?: number;164    firstTokenLatencyS?: number;165    metrics?: Record<string, unknown>;166    error?: string | null;167  } = {},168): void {169  const payload: CaseResult = {170    case_id: caseId,171    mode: options.mode ?? MODE,172    is_warmup: options.isWarmup ?? false,173    observation_summary: observationSummary,174    result_flag: resultFlag,175    metrics: options.metrics ?? {},176    error: options.error ?? null,177  };178 179  if (options.totalLatencyS !== undefined) {180    payload.total_latency_s = options.totalLatencyS;181  }182  if (options.firstTokenLatencyS !== undefined) {183    payload.first_token_latency_s = options.firstTokenLatencyS;184  }185 186  RESULTS.push(payload);187  emit('case_result', payload as Record<string, unknown>);188}189 190function summarizeResults(): Record<string, unknown> {191  const cases = new Map<string, CaseResult[]>();192  for (const result of RESULTS) {193    const existing = cases.get(result.case_id) ?? [];194    existing.push(result);195    cases.set(result.case_id, existing);196  }197 198  const summarizedCases = Object.fromEntries(199    [...cases.entries()].map(([caseId, items]) => {200      const measured = items.filter((item) => !item.is_warmup);201      const effectiveItems = measured.length > 0 ? measured : items;202      const totalLatencies = effectiveItems203        .map((item) => item.total_latency_s)204        .filter((value): value is number => typeof value === 'number');205      const firstTokenLatencies = effectiveItems206        .map((item) => item.first_token_latency_s)207        .filter((value): value is number => typeof value === 'number');208      const resultFlags = Object.fromEntries(209        [...new Set(effectiveItems.map((item) => item.result_flag))].map(210          (flag) => [211            flag,212            effectiveItems.filter((item) => item.result_flag === flag).length,213          ],214        ),215      );216      const median = (values: number[]): number | null => {217        if (values.length === 0) {218          return null;219        }220        const sorted = [...values].sort((a, b) => a - b);221        const middle = Math.floor(sorted.length / 2);222        if (sorted.length % 2 === 1) {223          return sorted[middle];224        }225        return (sorted[middle - 1] + sorted[middle]) / 2;226      };227 228      return [229        caseId,230        {231          mode: items.at(-1)?.mode ?? MODE,232          runs: effectiveItems.length,233          warmups: items.length - effectiveItems.length,234          result_flags: resultFlags,235          median_total_latency_s: median(totalLatencies),236          median_first_token_latency_s: median(firstTokenLatencies),237          observations: effectiveItems238            .slice(0, 3)239            .map((item) => item.observation_summary),240        },241      ];242    }),243  );244 245  const overallResultFlags = Object.fromEntries(246    [...new Set(RESULTS.map((item) => item.result_flag))].map((flag) => [247      flag,248      RESULTS.filter((item) => item.result_flag === flag).length,249    ]),250  );251 252  return {253    scenario: SCENARIO,254    run_label: RUN_LABEL,255    mode: MODE,256    result_count: RESULTS.length,257    cases: summarizedCases,258    result_flags: overallResultFlags,259  };260}261 262function finalize(exitCode: number): void {263  const metadataPayload = {264    exit_code: exitCode,265    runtime_context: runtimeContext(),266  };267  const summaryPayload = summarizeResults();268  emit('summary', {269    metadata: metadataPayload,270    summary: summaryPayload,271  });272 273  const directory = outputDir();274  if (!directory) {275    return;276  }277 278  mkdirSync(directory, { recursive: true });279  const metadataPath = join(directory, 'metadata.json');280  const resultsPath = join(directory, 'results.json');281  const summaryPath = join(directory, 'summary.json');282  writeJson(metadataPath, metadataPayload);283  writeJson(resultsPath, RESULTS);284  writeJson(summaryPath, summaryPayload);285  emit('artifact_paths', {286    metadata_path: metadataPath,287    results_path: resultsPath,288    summary_path: summaryPath,289  });290}291 292async function main(): Promise<number> {293  const caseId = process.env.PROBE_CASE_ID ?? 'case-0001';294  emit('banner', { context: runtimeContext() });295  startCase(caseId);296 297  // Replace this block with the narrow runtime question you want to test.298  // Example:299  // const core = await importRepoModule('packages/agents-core/src/index.ts');300  // const hasRunner = typeof (core as { Runner?: unknown }).Runner === 'function';301  // recordCaseResult(302  //   caseId,303  //   hasRunner304  //     ? 'Loaded agents-core source from the repository root.'305  //     : 'agents-core source loaded but expected exports were missing.',306  //   hasRunner ? 'expected' : 'negative',307  //   { metrics: { import_path: 'packages/agents-core/src/index.ts' } },308  // );309  recordCaseResult(310    caseId,311    'Template executed. Replace the placeholder block with the runtime behavior you want to observe.',312    'expected',313  );314 315  finalize(0);316  return 0;317}318 319void main().catch((error: unknown) => {320  const message =321    error instanceof Error ? `${error.name}: ${error.message}` : String(error);322  emit('fatal', { error: message });323  finalize(1);324  process.exitCode = 1;325});326