scripts/fetch-codex-manual.mjs
scripts/fetch-codex-manual.mjsBrowse 17 files
4,088 tokens
16,085 bytes
Token encoding: o200k_base
Snapshot 18d7ace
← Back to SKILL.md
1#!/usr/bin/env node2import {3 access,4 mkdir,5 readFile,6 rename,7 rm,8 stat,9 writeFile,10} from "node:fs/promises";11import { constants as fsConstants } from "node:fs";12import { execFile } from "node:child_process";13import { createHash } from "node:crypto";14import path from "node:path";15import process from "node:process";16import { pathToFileURL } from "node:url";17import { inspect, promisify } from "node:util";18 19const DEFAULT_MANUAL_URL = "https://developers.openai.com/codex/codex-manual.md";20const DEFAULT_CACHE_DIR_NAME = "openai-docs-cache";21const CACHE_FILE_NAME = "codex-manual.md";22const OUTLINE_FILE_NAME = "codex-manual.outline.md";23const HASH_HEADER = "x-content-sha256";24const USER_AGENT = "codex-openai-docs";25const execFileAsync = promisify(execFile);26 27class ManualFetchError extends Error {28 constructor(message, options) {29 super(message, options);30 this.name = "ManualFetchError";31 }32}33 34const sha256 = (value) => createHash("sha256").update(value).digest("hex");35 36const withTimeout = async (promiseFactory, timeoutMs) => {37 const controller = new AbortController();38 const timeout = setTimeout(() => controller.abort(), timeoutMs);39 try {40 return await promiseFactory(controller.signal);41 } finally {42 clearTimeout(timeout);43 }44};45 46const proxyConfigured = () =>47 process.env.HTTP_PROXY ||48 process.env.HTTPS_PROXY ||49 process.env.http_proxy ||50 process.env.https_proxy;51 52const responseHeaders = (headers) => ({53 get(name) {54 return headers.get(name.toLowerCase()) ?? null;55 },56});57 58const makeResponse = ({ body, headers, status }) => ({59 headers: responseHeaders(headers),60 ok: status >= 200 && status < 300,61 status,62 async text() {63 return body;64 },65});66 67const parseCurlHeaders = (rawHeaders) => {68 const normalized = rawHeaders.replace(/\r\n/g, "\n").trim();69 const blocks = normalized.split(/\n\n+/).filter(Boolean);70 const headerBlock = [...blocks]71 .reverse()72 .find((block) => block.startsWith("HTTP/"));73 74 if (!headerBlock) {75 throw new ManualFetchError("curl did not return HTTP response headers.");76 }77 78 const [statusLine, ...lines] = headerBlock.split("\n");79 const statusMatch = /^HTTP\/\S+\s+(\d{3})/.exec(statusLine);80 if (!statusMatch) {81 throw new ManualFetchError(82 `Could not parse HTTP status from curl response: ${statusLine}`83 );84 }85 86 const headers = new Map();87 lines.forEach((line) => {88 const separator = line.indexOf(":");89 if (separator === -1) return;90 const name = line.slice(0, separator).trim().toLowerCase();91 const value = line.slice(separator + 1).trim();92 headers.set(name, value);93 });94 95 return {96 headers,97 status: Number(statusMatch[1]),98 };99};100 101const tempFilePath = (cacheDir, suffix) =>102 path.join(103 cacheDir,104 `.fetch-codex-manual-${process.pid}-${Date.now()}-${Math.random()105 .toString(16)106 .slice(2)}${suffix}`107 );108 109const requestManualWithCurl = async (url, { cacheDir, method, timeoutMs }) => {110 const headerPath = tempFilePath(cacheDir, ".headers");111 const bodyPath = tempFilePath(cacheDir, ".body");112 const curlNames =113 process.platform === "win32" ? ["curl.exe", "curl"] : ["curl"];114 const args = [115 "--silent",116 "--show-error",117 "--location",118 "--dump-header",119 headerPath,120 "--output",121 bodyPath,122 "--user-agent",123 USER_AGENT,124 "--max-time",125 String(Math.max(1, Math.ceil(timeoutMs / 1000))),126 ];127 128 if (method === "HEAD") {129 args.push("--head");130 } else {131 args.push("--request", method);132 }133 args.push(url);134 135 let lastError;136 for (const curlName of curlNames) {137 try {138 await execFileAsync(curlName, args, { windowsHide: true });139 const [rawHeaders, body] = await Promise.all([140 readFile(headerPath, "utf8"),141 readFile(bodyPath, "utf8"),142 ]);143 const { headers, status } = parseCurlHeaders(rawHeaders);144 return makeResponse({ body, headers, status });145 } catch (error) {146 lastError = error;147 if (error?.code !== "ENOENT") break;148 } finally {149 await Promise.all([150 rm(headerPath, { force: true }),151 rm(bodyPath, { force: true }),152 ]);153 }154 }155 156 if (lastError?.code === "ENOENT") {157 throw new ManualFetchError("curl is unavailable in this environment.", {158 cause: lastError,159 });160 }161 throw new ManualFetchError(`${method} ${url} could not be fetched.`, {162 cause: lastError,163 });164};165 166const requestManualWithFetch = async (url, { method, timeoutMs }) => {167 if (typeof fetch !== "function") {168 throw new ManualFetchError(169 "Native fetch is unavailable in this Node runtime."170 );171 }172 173 return withTimeout(174 (signal) =>175 fetch(url, {176 method,177 headers: { "User-Agent": USER_AGENT },178 signal,179 }),180 timeoutMs181 );182};183 184const requestManual = async (url, { cacheDir, method, timeoutMs }) => {185 const preferCurl = Boolean(proxyConfigured()) || typeof fetch !== "function";186 const transports = preferCurl187 ? [188 () => requestManualWithCurl(url, { cacheDir, method, timeoutMs }),189 () => requestManualWithFetch(url, { method, timeoutMs }),190 ]191 : [192 () => requestManualWithFetch(url, { method, timeoutMs }),193 () => requestManualWithCurl(url, { cacheDir, method, timeoutMs }),194 ];195 196 let lastError;197 for (const transport of transports) {198 try {199 const response = await transport();200 if (!response.ok) {201 throw new ManualFetchError(202 `${method} ${url} failed with HTTP ${response.status}.`203 );204 }205 return response;206 } catch (error) {207 lastError = error;208 }209 }210 211 throw new ManualFetchError(`${method} ${url} could not be fetched.`, {212 cause: lastError,213 });214};215 216const readHeaderSha = (response) => {217 const value = response.headers.get(HASH_HEADER);218 if (!value || !/^[a-f0-9]{64}$/i.test(value)) {219 throw new ManualFetchError(`Manual response is missing ${HASH_HEADER}.`);220 }221 return value.toLowerCase();222};223 224const nearestExistingParent = async (target) => {225 let current = target;226 while (true) {227 try {228 const info = await stat(current);229 return info.isDirectory() ? current : null;230 } catch (error) {231 if (error?.code !== "ENOENT") return null;232 }233 234 const parent = path.dirname(current);235 if (parent === current) return null;236 current = parent;237 }238};239 240const usableCacheDir = async (cacheDir) => {241 if (!cacheDir) return null;242 const resolved = path.resolve(cacheDir);243 244 try {245 const info = await stat(resolved);246 if (!info.isDirectory()) return null;247 } catch (error) {248 if (error?.code !== "ENOENT") return null;249 }250 251 const parent = await nearestExistingParent(resolved);252 if (!parent) return null;253 254 try {255 await access(parent, fsConstants.W_OK | fsConstants.X_OK);256 } catch {257 return null;258 }259 260 return resolved;261};262 263const defaultCacheDirCandidates = () => {264 const candidates = [];265 const seen = new Set();266 const pushCandidate = (candidate) => {267 if (!candidate || seen.has(candidate)) return;268 seen.add(candidate);269 candidates.push(candidate);270 };271 272 [process.env.TMPDIR, process.env.TEMP, process.env.TMP].forEach((baseDir) => {273 if (baseDir) {274 pushCandidate(path.join(baseDir, DEFAULT_CACHE_DIR_NAME));275 }276 });277 278 if (process.platform !== "win32") {279 pushCandidate(`/private/tmp/${DEFAULT_CACHE_DIR_NAME}`);280 pushCandidate(`/tmp/${DEFAULT_CACHE_DIR_NAME}`);281 }282 283 return candidates;284};285 286const resolveCacheDir = async (cacheDir) => {287 if (cacheDir) {288 return usableCacheDir(cacheDir);289 }290 291 for (const candidate of defaultCacheDirCandidates()) {292 const usable = await usableCacheDir(candidate);293 if (usable) return usable;294 }295 296 return null;297};298 299const cacheFilePath = (cacheDir) => path.join(cacheDir, CACHE_FILE_NAME);300 301const outlineFilePath = (cacheDir) => path.join(cacheDir, OUTLINE_FILE_NAME);302 303const manualLines = (manual) => {304 const lines = manual.replace(/\r\n/g, "\n").split("\n");305 if (lines[lines.length - 1] === "") lines.pop();306 return lines;307};308 309const sectionTitle = (rawTitle) =>310 rawTitle.replace(/\s+#+\s*$/, "").replace(/\s+/g, " ").trim();311 312const buildOutline = (manual) => {313 const lines = manualLines(manual);314 const headings = [];315 let inFence = false;316 317 lines.forEach((line, index) => {318 if (/^\s*(```|~~~)/.test(line)) {319 inFence = !inFence;320 return;321 }322 if (inFence) return;323 324 const match = /^(#{1,6})\s+(.+?)\s*$/.exec(line);325 if (!match) return;326 327 const level = match[1].length;328 if (level < 2 || level > 3) return;329 330 headings.push({331 level,332 title: sectionTitle(match[2]),333 startLine: index + 1,334 endLine: lines.length,335 });336 });337 338 for (let index = 0; index < headings.length; index += 1) {339 const heading = headings[index];340 const nextPeer = headings341 .slice(index + 1)342 .find((candidate) => candidate.level <= heading.level);343 if (nextPeer) {344 heading.endLine = nextPeer.startLine - 1;345 }346 }347 348 if (headings.length === 0) {349 return {350 headingCount: 0,351 lineCount: lines.length,352 text: "No markdown headings found.",353 };354 }355 356 const minLevel = Math.min(...headings.map((heading) => heading.level));357 return {358 headingCount: headings.length,359 lineCount: lines.length,360 text: headings361 .map((heading) => {362 const indent = " ".repeat(heading.level - minLevel);363 return `${indent}- ${heading.title} (lines ${heading.startLine}-${heading.endLine})`;364 })365 .join("\n"),366 };367};368 369const outlineMarkdown = (outline) => `# Codex Manual Outline\n\n${outline.text}\n`;370 371const manualStatusLine = (status) =>372 status.cacheStatus === "hit"373 ? "Manual status: local manual was already current."374 : "Manual status: local manual was updated.";375 376const formatResult = ({ status, outlineText }) =>377 [378 `Manual path: ${status.manualPath}`,379 `Outline path: ${status.outlinePath}`,380 manualStatusLine(status),381 "",382 outlineText,383 ].join("\n");384 385const readCachedManual = async (cacheDir, expectedSha256) => {386 try {387 const manual = await readFile(cacheFilePath(cacheDir), "utf8");388 return sha256(manual) === expectedSha256 ? manual : null;389 } catch {390 return null;391 }392};393 394const writeCachedManual = async (cacheDir, manual) => {395 await mkdir(cacheDir, { recursive: true });396 const tmpPath = tempFilePath(cacheDir, `.${CACHE_FILE_NAME}.tmp`);397 await writeFile(tmpPath, manual, "utf8");398 await rename(tmpPath, cacheFilePath(cacheDir));399};400 401const writeOutline = async (cacheDir, outlineText) => {402 await mkdir(cacheDir, { recursive: true });403 const tmpPath = tempFilePath(cacheDir, `.${OUTLINE_FILE_NAME}.tmp`);404 await writeFile(tmpPath, outlineText, "utf8");405 await rename(tmpPath, outlineFilePath(cacheDir));406};407 408const fetchCodexManual = async ({409 manualUrl = DEFAULT_MANUAL_URL,410 cacheDir,411 timeoutMs = 30000,412} = {}) => {413 const resolvedCacheDir = await resolveCacheDir(cacheDir);414 if (!resolvedCacheDir) {415 throw new ManualFetchError(416 "Manual cache directory is unavailable; pass --cache-dir to override or use OpenAI Docs MCP fallback."417 );418 }419 await mkdir(resolvedCacheDir, { recursive: true });420 421 const headResponse = await requestManual(manualUrl, {422 cacheDir: resolvedCacheDir,423 method: "HEAD",424 timeoutMs,425 });426 const expectedSha256 = readHeaderSha(headResponse);427 const manualPath = cacheFilePath(resolvedCacheDir);428 const outlinePath = outlineFilePath(resolvedCacheDir);429 const checkedAt = new Date().toISOString();430 431 const cachedManual = await readCachedManual(resolvedCacheDir, expectedSha256);432 if (cachedManual !== null) {433 const outline = buildOutline(cachedManual);434 const outlineText = outlineMarkdown(outline);435 await writeOutline(resolvedCacheDir, outlineText);436 437 return {438 outlineText,439 status: {440 manualUrl,441 headerSha256: expectedSha256,442 fetchedManualSha256: expectedSha256,443 manualHashMatches: true,444 cacheStatus: "hit",445 cacheDir: resolvedCacheDir,446 manualPath,447 outlinePath,448 checkedAt,449 lineCount: outline.lineCount,450 headingCount: outline.headingCount,451 },452 };453 }454 455 const getResponse = await requestManual(manualUrl, {456 cacheDir: resolvedCacheDir,457 method: "GET",458 timeoutMs,459 });460 const getHeaderSha256 = readHeaderSha(getResponse);461 if (getHeaderSha256 !== expectedSha256) {462 throw new ManualFetchError(463 `${HASH_HEADER} changed between HEAD and GET for ${manualUrl}.`464 );465 }466 467 const manualText = await getResponse.text();468 const actualSha256 = sha256(manualText);469 const manualHashMatches = actualSha256 === expectedSha256;470 if (!manualHashMatches) {471 throw new ManualFetchError(472 `${HASH_HEADER} did not match the fetched manual body for ${manualUrl}.`473 );474 }475 476 await writeCachedManual(resolvedCacheDir, manualText);477 const outline = buildOutline(manualText);478 const outlineText = outlineMarkdown(outline);479 await writeOutline(resolvedCacheDir, outlineText);480 481 return {482 outlineText,483 status: {484 manualUrl,485 headerSha256: expectedSha256,486 fetchedManualSha256: actualSha256,487 manualHashMatches,488 cacheStatus: "updated",489 cacheDir: resolvedCacheDir,490 manualPath,491 outlinePath,492 checkedAt,493 lineCount: outline.lineCount,494 headingCount: outline.headingCount,495 },496 };497};498 499const parseArgs = (argv) => {500 const args = {501 manualUrl: DEFAULT_MANUAL_URL,502 cacheDir: undefined,503 timeoutMs: 30000,504 statusJson: false,505 };506 507 for (let index = 0; index < argv.length; index += 1) {508 const arg = argv[index];509 if (arg === "--manual-url") {510 args.manualUrl = argv[++index];511 } else if (arg === "--cache-dir") {512 args.cacheDir = argv[++index];513 } else if (arg === "--timeout-ms") {514 args.timeoutMs = Number(argv[++index]);515 } else if (arg === "--status-json") {516 args.statusJson = true;517 } else {518 throw new ManualFetchError(`Unknown argument: ${arg}`);519 }520 }521 522 if (!args.manualUrl) {523 throw new ManualFetchError("--manual-url cannot be empty.");524 }525 if (!Number.isFinite(args.timeoutMs) || args.timeoutMs <= 0) {526 throw new ManualFetchError("--timeout-ms must be a positive number.");527 }528 529 return args;530};531 532const main = async () => {533 const args = parseArgs(process.argv.slice(2));534 const { outlineText, status } = await fetchCodexManual(args);535 536 process.stdout.write(formatResult({ status, outlineText }));537 538 if (args.statusJson) {539 console.error(JSON.stringify(status));540 }541};542 543const envProxyHint = () => {544 if (proxyConfigured()) {545 return "Hint: proxy env vars are present. This helper prefers `curl` in proxied sessions; if requests still fail, verify `curl` is installed and the proxy configuration is valid.";546 }547 if (typeof fetch !== "function") {548 return "Hint: native fetch is unavailable in this Node runtime. Install `curl` or use a newer Node version to fetch the manual.";549 }550 if (process.platform === "win32") {551 return "Hint: on Windows, pass a cache dir under `%TEMP%` or `%TMP%`.";552 }553 return null;554};555 556const formatErrorDetails = (error) => {557 const details = inspect(error, {558 breakLength: 120,559 colors: false,560 compact: false,561 depth: 8,562 });563 if (!error?.cause) {564 return details;565 }566 567 return `${details}\n\nCause:\n${inspect(error.cause, {568 breakLength: 120,569 colors: false,570 compact: false,571 depth: 8,572 })}`;573};574 575const isCliEntrypoint = () => {576 const entrypoint = process.argv[1];577 if (!entrypoint) {578 return false;579 }580 581 return pathToFileURL(entrypoint).href === import.meta.url;582};583 584if (isCliEntrypoint()) {585 main().catch((error) => {586 console.error(`Error: ${error.message}`);587 const hint = envProxyHint();588 if (hint) {589 console.error(hint);590 }591 console.error("");592 console.error("Details:");593 console.error(formatErrorDetails(error));594 process.exitCode = 1;595 });596}597 598export { DEFAULT_MANUAL_URL, fetchCodexManual };599 