auteur

1#!/usr/bin/env node2/**3 * refscout.mjs — reference scouting: find live award-level sites, then take them apart.4 *5 * Usage:6 *   node scripts/refscout.mjs <url> [<url>...]        analyse specific sites7 *   node scripts/refscout.mjs --from awwwards          harvest the current gallery8 *   node scripts/refscout.mjs --from awwwards:scrolling --limit 69 *   node scripts/refscout.mjs --from awwwards --search "coffee brand" --limit 510 *11 * Options:12 *   --limit N       how many sites (default 6, hard cap 15)13 *   --out DIR       output dir (default design/refs)14 *   --shots N       screenshots per site (default 3: hero + 2 scroll stops)15 *   --no-shots      fingerprint only, no browser screenshots16 *   --width N       viewport width for shots (default 1440)17 *18 * Writes DIR/REFERENCES.md (read this), DIR/refs.json, DIR/shots/*.png.19 * Exits 0 if at least one site was profiled, 1 otherwise.20 */21 22import { mkdir, writeFile } from 'fs/promises';23import { resolve, join } from 'path';24 25const args = process.argv.slice(2);26if (!args.length || args.includes('--help')) {27  console.log(`refscout.mjs — scout live website references and fingerprint their mechanics28 29  node scripts/refscout.mjs <url> [<url>...]30  node scripts/refscout.mjs --from awwwards[:<tag>] [--search "<text>"] [--limit 6]31 32  --limit N  --out DIR  --shots N  --no-shots  --width N`);33  process.exit(0);34}35const get = (f, d) => { const i = args.indexOf(f); return i !== -1 ? args[i + 1] : d; };36const has = f => args.includes(f);37 38const limit    = Math.min(parseInt(get('--limit', '6'), 10) || 6, 15);39const outDir   = resolve(get('--out', 'design/refs'));40const shotsN   = has('--no-shots') ? 0 : (parseInt(get('--shots', '3'), 10) || 3);41const width    = parseInt(get('--width', '1440'), 10) || 1440;42const from     = get('--from', null);43const search   = get('--search', null);44const urls     = args.filter(a => /^https?:\/\//.test(a));45 46let chromium;47try { ({ chromium } = await import('playwright')); }48catch { console.error('playwright not found. Install: npm i -D playwright && npx playwright install chromium'); process.exit(2); }49 50const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';51 52/**53 * Award-level sites are heavy streaming-SSR / WebGL pages: 'load' and 'networkidle'54 * frequently never fire. Commit + a timed settle is the only navigation that survives them.55 * Returns false when DOMContentLoaded never fired — those pages need freeze() before capture.56 */57async function open(page, url, settle = 7000, onStatus) {58  const resp = await page.goto(url, { waitUntil: 'commit', timeout: 30_000 });59  onStatus?.(resp?.status?.() ?? 0);60  const dcl = await page.waitForLoadState('domcontentloaded', { timeout: 12_000 }).then(() => true).catch(() => false);61  await page.waitForTimeout(dcl ? settle : settle + 5000); // no DCL → give hydration longer62  return dcl;63}64 65/**66 * A page whose HTML stream never closes never yields a compositor frame either, so every67 * screenshot on it times out (measured on basement.studio and lusion.co). `window.stop()`68 * ends the stream and capture drops to ~40ms. Call it only AFTER the settle wait — stopping69 * early aborts the CSS/JS the fingerprint needs and leaves you reading Times-New-Roman70 * defaults off a half-loaded page.71 */72async function freeze(page, dcl) {73  if (!dcl) await page.evaluate(() => window.stop()).catch(() => {});74}75 76// --- library signatures, matched against the concatenated JS the page actually loaded ---77// Bundlers hide globals (window.gsap is empty on virtually every modern site), but78// minifiers keep string literals and class names, so the bundle text still tells the truth.79const LIBS = [80  ['GSAP',            /gsap\.registerPlugin|GreenSock|_gsap|gsap\.timeline|gsap\.to\(/],81  ['ScrollTrigger',   /ScrollTrigger/],82  ['ScrollSmoother',  /ScrollSmoother/],83  ['SplitText',       /SplitText|SplitType/],84  ['Lenis',           /\blenis\b|new Lenis|lenis-smooth/i],85  ['Locomotive',      /locomotive-scroll|LocomotiveScroll|has-scroll-smooth/],86  ['three.js',        /THREE\.WebGLRenderer|WebGLRenderer|three\.module|\bTHREE\b/],87  ['R3F',             /react-three|@react-three\/fiber|useFrame/],88  ['OGL',             /\bogl\b|new Renderer\(\{.*dpr/],89  ['PixiJS',          /PIXI\.|pixi\.js/],90  ['Framer Motion',   /framer-motion|motion-dom|framerAppearId/],91  ['Motion One',      /@motionone|motion\.dev/],92  ['Barba',           /barba\.init|@barba|new Barba/],93  ['Swup',            /new Swup|@swup/],94  ['Matter.js',       /Matter\.Engine/],95  ['Rive',            /@rive-app|rive\.wasm/],96  ['Lottie',          /lottie-web|bodymovin/],97  ['Spline',          /splinetool|spline-viewer/],98  ['Theatre.js',      /@theatre\/core/],99  ['Swiper',          /new Swiper|swiper-bundle/],100  ['Splitting',       /Splitting\(/],101];102 103function classifyPeak(f) {104  const out = [];105  if (f.videos.some(v => !v.autoplay && v.preload !== 'none') && f.scrollRatio > 4) out.push('likely scroll-scrubbed video');106  if (f.webgl && f.scrollRatio > 3) out.push('WebGL scene driven by scroll');107  else if (f.webgl) out.push('WebGL hero');108  if (f.pinSpacers > 0) out.push(`${f.pinSpacers} pinned scene(s) (GSAP ScrollTrigger)`);109  if (f.stickies > 2) out.push(`${f.stickies} sticky layers (stack/pin effect)`);110  if (f.cssTimeline) out.push('CSS scroll-driven animations (animation-timeline)');111  if (f.libs.includes('Lenis') || f.libs.includes('Locomotive') || f.libs.includes('ScrollSmoother')) out.push('smoothed/virtualised scroll');112  if (f.mixBlend > 0) out.push(`${f.mixBlend} mix-blend-mode layer(s)`);113  if (f.cursorNone) out.push('custom cursor');114  if (f.videos.length && !out.length) out.push('looping video texture');115  return out;116}117 118async function fingerprint(ctx, url) {119  const page = await ctx.newPage();120  const js = [];121  let jsBytes = 0;122  page.on('response', async res => {123    if (jsBytes > 4_000_000) return;124    const ct = res.headers()['content-type'] || '';125    if (!/javascript|ecmascript/.test(ct)) return;126    try { const t = await res.text(); jsBytes += t.length; js.push(t); } catch {}127  });128 129  let httpStatus = 0;130  try {131    const dcl = await open(page, url, 8000, s => { httpStatus = s; });132    // one scroll pass so pinning / lazy scenes actually initialise before we look133    await page.evaluate(() => window.scrollTo({ top: innerHeight * 1.6, behavior: 'instant' }));134    await page.waitForTimeout(2500);135 136    const dom = await page.evaluate(() => {137      const cs = getComputedStyle;138      // A blocked / challenge / download navigation can leave us with no body at all.139      // Bail with an explicitly empty reading rather than throwing away the whole site.140      if (!document.body) return { title: (document.title || '').slice(0, 90), sheets: 0, readyState: document.readyState, globals: [], htmlClass: '', pinSpacers: 0, stickies: 0, canvases: 0, webgl: false, videos: [], mixBlend: 0, cursorNone: false, cssTimeline: false, fontFaces: [], display: null, bodyFont: '', bg: '', palette: [], sections: 0, scrollH: 0, vh: window.innerHeight, framework: 'unknown' };141      const all = [...document.querySelectorAll('*')].slice(0, 4000);142      const canvases = [...document.querySelectorAll('canvas')];143      let cssTimeline = false;144      const faces = new Set();145      for (const ss of document.styleSheets) {146        try {147          for (const r of ss.cssRules) {148            const t = r.cssText || '';149            if (t.includes('animation-timeline') || t.includes('view-timeline')) cssTimeline = true;150            if (r.style && r.constructor.name === 'CSSFontFaceRule') faces.add((r.style.fontFamily || '').replace(/["']/g, ''));151          }152        } catch {}153      }154      // Chromium reports modern colour syntaxes verbatim (`lab(48.5 0 0)`, `oklch(...)`), which is155      // unreadable as a palette. Round-trip every value through a canvas so the report shows a156      // colour a human can picture, keeping the original alongside.157      const cx = document.createElement('canvas').getContext('2d', { willReadFrequently: true });158      const toHex = v => {159        try {160          cx.clearRect(0, 0, 1, 1); cx.fillStyle = '#000'; cx.fillStyle = v;161          cx.fillRect(0, 0, 1, 1);162          const [r, g, b, a] = cx.getImageData(0, 0, 1, 1).data;163          const hex = '#' + [r, g, b].map(n => n.toString(16).padStart(2, '0')).join('');164          return a < 250 ? `${hex}@${(a / 255).toFixed(2)}` : hex;165        } catch { return null; }166      };167      // sample the real palette from what is painted, not from the token file168      const colors = {};169      for (const el of all) {170        const s = cs(el);171        for (const c of [s.backgroundColor, s.color]) {172          if (!c || c === 'rgba(0, 0, 0, 0)') continue;173          colors[c] = (colors[c] || 0) + 1;174        }175      }176      const palette = Object.entries(colors).sort((a, b) => b[1] - a[1]).slice(0, 6).map(([c, n]) => {177        const hex = toHex(c);178        return hex && !/^(#|rgb)/.test(c) ? `${hex} (${c}) ×${n}` : `${hex || c} ×${n}`;179      });180      // The display face is the biggest thing actually painted, not the first h1 — a hidden or181      // fallback-styled heading reports whatever the CSS cascade left there and will happily182      // claim an awwwards winner ships Inter. Measure instead: largest visible rendered text.183      let display = null, bestPx = 0;184      for (const e of all) {185        if (e.children.length || (e.textContent || '').trim().length < 2) continue;186        const s = cs(e);187        if (s.visibility === 'hidden' || s.display === 'none' || +s.opacity === 0) continue;188        const r = e.getBoundingClientRect();189        if (r.width < 4 || r.height < 4) continue;190        const px = parseFloat(s.fontSize) || 0;191        if (px > bestPx) { bestPx = px; display = { font: s.fontFamily.slice(0, 60), px: Math.round(px), weight: s.fontWeight, sample: e.textContent.trim().replace(/\s+/g, ' ').slice(0, 28) }; }192      }193      return {194        title: (document.title || '').slice(0, 90),195        sheets: document.styleSheets.length,196        readyState: document.readyState,197        globals: ['gsap', 'ScrollTrigger', 'Lenis', 'THREE', 'locomotiveScroll', 'barba', 'Swup', 'PIXI'].filter(k => k in window),198        htmlClass: document.documentElement.className.slice(0, 100),199        pinSpacers: document.querySelectorAll('.pin-spacer').length,200        stickies: all.filter(e => cs(e).position === 'sticky').length,201        canvases: canvases.length,202        webgl: canvases.some(c => { try { return !!(c.getContext('webgl2') || c.getContext('webgl')); } catch { return false; } }),203        videos: [...document.querySelectorAll('video')].slice(0, 4).map(v => ({204          autoplay: v.autoplay, loop: v.loop, muted: v.muted, preload: v.preload,205          src: (v.currentSrc || v.src || '').split('/').pop().slice(0, 40),206        })),207        mixBlend: all.filter(e => cs(e).mixBlendMode !== 'normal').length,208        cursorNone: cs(document.body).cursor === 'none',209        cssTimeline,210        fontFaces: [...faces].filter(Boolean).slice(0, 8),211        display,212        bodyFont: cs(document.body).fontFamily.slice(0, 70),213        bg: cs(document.body).backgroundColor,214        palette,215        sections: document.querySelectorAll('section, main > div, [class*="section"]').length,216        scrollH: document.documentElement.scrollHeight,217        vh: window.innerHeight,218        framework:219          document.querySelector('[data-framer-name],[data-framer-root]') ? 'Framer' :220          document.querySelector('[data-wf-page]') ? 'Webflow' :221          document.querySelector('#__next,[id^=__next]') ? 'Next.js' :222          document.querySelector('astro-island,[data-astro-cid]') ? 'Astro' :223          document.querySelector('[data-sveltekit-preload-data]') ? 'SvelteKit' :224          document.querySelector('#app,[data-v-app]') ? 'Vue/Nuxt' :225          document.querySelector('[data-shopify],[id^=shopify]') ? 'Shopify' : 'unknown',226      };227    });228 229    const blob = js.join('\n');230    const libs = LIBS.filter(([, re]) => re.test(blob)).map(([n]) => n);231    for (const g of dom.globals) if (!libs.some(l => l.toLowerCase().includes(g.toLowerCase().slice(0, 5)))) libs.push(`${g} (global)`);232 233    const f = { url, ...dom, libs, jsBytes, scrollRatio: +(dom.scrollH / Math.max(dom.vh, 1)).toFixed(1) };234    f.mechanics = classifyPeak(f);235    // Some sites (Vercel-edge streamers, bot-walled studios) hand a headless client the SSR236    // HTML and then never deliver CSS or JS. Everything we'd read off that page — fonts,237    // palette, libs — is browser default, i.e. a confident lie. Detect it and say so.238    // Chrome's unstyled body is Times New Roman with zero web fonts loaded; a real site that239    // merely leaves body at the default still has @font-face rules from its stylesheet.240    const unstyled = /^"?Times New Roman/.test(dom.bodyFont) && !dom.fontFaces.length;241    // An error page has a title, a palette and a page shape, and will happily be written up as a242    // reference with a blank `steal:` line waiting to be filled. It is not a design.243    f.httpStatus = httpStatus;244    f.errorPage = httpStatus >= 400 || /^\s*(4\d\d|5\d\d)|bad gateway|not found|forbidden|service unavailable|internal server error/i.test(dom.title || '');245    f.thin = dom.sheets === 0 || unstyled || f.errorPage;246 247    // Screenshots are best-effort on purpose: a capture that fails must never throw away248    // a fingerprint we already paid for.249    f.shots = [];250    // A site that never got its CSS has nothing to photograph: take one frame as evidence the251    // capture failed, not a full journey of identical unstyled pages.252    const n = f.thin ? Math.min(1, shotsN) : shotsN;253    if (n > 0) {254      await freeze(page, dcl);255      const slug = url.replace(/^https?:\/\//, '').replace(/[^\w.-]+/g, '_').slice(0, 40);256      const max = Math.max(0, dom.scrollH - dom.vh);257      for (let i = 0; i < n; i++) {258        const y = n === 1 ? 0 : Math.round((i / (n - 1)) * max);259        const name = `${slug}-${String(i).padStart(2, '0')}.png`;260        try {261          await page.evaluate(t => window.scrollTo({ top: t, behavior: 'instant' }), y);262          await page.waitForTimeout(1600);263          await page.screenshot({ path: join(outDir, 'shots', name), timeout: 20_000 });264          f.shots.push(`shots/${name}`);265        } catch (e) {266          (f.shotErrors ??= []).push(`stop ${i}: ${e.message.split('\n')[0]}`);267        }268      }269      f.shotsWanted = n;270    }271    return f;272  } catch (e) {273    return { url, error: e.message.split('\n')[0] };274  } finally {275    await page.close();276  }277}278 279// --- gallery harvesting -------------------------------------------------------280// awwwards is the one gallery whose listing AND detail pages both render reliably headless281// and expose the outbound site URL. Other galleries either need JS we can't wait out or bury282// the real URL behind affiliate redirects — for those, find URLs with web_search and pass them283// positionally.284async function harvestAwwwards(ctx, tag, text, n) {285  const page = await ctx.newPage();286  const base = 'https://www.awwwards.com/websites/';287  const url = text ? `${base}?text=${encodeURIComponent(text)}` : tag ? `${base}${tag}/` : base;288  const found = [];289  try {290    await open(page, url, 6000);291    const slugs = await page.evaluate(() =>292      [...new Set([...document.querySelectorAll('a[href*="/sites/"]')].map(a => a.getAttribute('href')).filter(h => h && h.startsWith('/sites/')))]);293    if (!slugs.length) console.error(`[refscout] awwwards listing returned no cards for ${url}`);294    for (const slug of slugs.slice(0, n)) {295      try {296        await open(page, 'https://www.awwwards.com' + slug, 3500);297        const d = await page.evaluate(() => {298          const visit = [...document.querySelectorAll('a[href^="http"]')]299            .find(a => /visit site/i.test(a.innerText) || /button/.test(a.className));300          const ext = visit?.href || [...document.querySelectorAll('a[href^="http"]')]301            .map(a => a.href).find(h => !/awwwards|twitter|x\.com|facebook|instagram|linkedin|behance|dribbble/i.test(h));302          return {303            site: ext,304            name: (document.querySelector('h1')?.innerText || document.title).trim().slice(0, 60),305          };306        });307        if (d.site) found.push({ ...d, via: 'awwwards' + slug });308      } catch {}309    }310  } catch (e) {311    console.error(`[refscout] awwwards harvest failed: ${e.message.split('\n')[0]}`);312  } finally { await page.close(); }313  return found;314}315 316// --- run ----------------------------------------------------------------------317await mkdir(join(outDir, 'shots'), { recursive: true });318const browser = await chromium.launch({ headless: true });319const ctx = await browser.newContext({ userAgent: UA, viewport: { width, height: 900 }, ignoreHTTPSErrors: true });320 321let targets = urls.map(u => ({ site: u, name: null, via: 'direct' }));322if (from) {323  const [gallery, tag] = from.split(':');324  if (gallery === 'awwwards') targets = targets.concat(await harvestAwwwards(ctx, tag, search, limit));325  else console.error(`[refscout] unknown gallery "${gallery}" — supported: awwwards. Pass URLs directly instead (find them with web_search).`);326}327targets = targets.slice(0, limit);328 329if (!targets.length) {330  console.error('[refscout] no targets. Pass URLs, or --from awwwards.');331  await browser.close();332  process.exit(1);333}334 335const results = [];336for (const t of targets) {337  process.stderr.write(`[refscout] profiling ${t.site} …\n`);338  const f = await fingerprint(ctx, t.site);339  results.push({ ...t, ...f });340}341await browser.close();342 343// --- report -------------------------------------------------------------------344const ok = results.filter(r => !r.error);345const lines = [];346lines.push('# REFERENCES — scouted ' + new Date().toISOString().slice(0, 10));347lines.push('');348lines.push('Mechanics, not skins. For each reference name the ONE idea you are taking and how it');349lines.push('changes for this brand. Copying a reference wholesale is slop with better taste.');350lines.push('');351lines.push('| # | site | stack detected | page shape | mechanics |');352lines.push('|---|---|---|---|---|');353ok.forEach((r, i) => {354  const cell = r.errorPage ? '_error page — not a design_' : r.thin ? '_no capture — see below_' : null;355  lines.push(`| ${i + 1} | [${r.name || r.url.replace(/^https?:\/\//, '')}](${r.url}) | ${cell ?? (r.libs.slice(0, 4).join(', ') || '—')} | ${cell ?? `${r.scrollRatio}× vh, ${r.sections} sections`} | ${cell ?? (r.mechanics.slice(0, 3).join('; ') || '—')} |`);356});357lines.push('');358 359ok.forEach((r, i) => {360  lines.push(`## ${i + 1}. ${r.name || r.title || r.url}`);361  lines.push(`- url: ${r.url}${r.via && r.via !== 'direct' ? `  ·  via ${r.via}` : ''}`);362  if (r.errorPage) {363    lines.push(`- ⚠ **NOT A DESIGN** — this URL returned an error page${r.httpStatus >= 400 ? ` (HTTP ${r.httpStatus})` : ''}, title "${r.title}". Nothing here is a reference. Drop it, or find the site's real address.`);364    lines.push('');365    return;366  }367  if (r.thin) {368    lines.push(`- ⚠ **NO CAPTURE** — this site served the headless browser bare HTML and never delivered its CSS/JS (${r.sheets} stylesheets, readyState \`${r.readyState}\`). Any font, colour or stack reading here would be the browser's defaults, so none is reported. Open the URL yourself, or describe it from the gallery write-up — and check the page title above, a dead link from the gallery lands here too.`);369    if (r.shots.length) lines.push(`- unstyled shots (evidence that the capture failed, not a look at the design): ${r.shots.map(s => `\`${s}\``).join(' ')}`);370    lines.push('');371    return;372  }373  lines.push(`- stack: ${r.libs.join(', ') || 'nothing detected'}${r.framework !== 'unknown' ? `  ·  built with ${r.framework}` : ''}`);374  lines.push(`- mechanics: ${r.mechanics.join('; ') || '—'}`);375  lines.push(`- page: ${r.scrollRatio}× viewport tall · ${r.sections} sections · ${r.canvases} canvas${r.webgl ? ' (WebGL live)' : ''} · ${r.videos.length} video`);376  lines.push(`- type: display \`${r.display ? `${r.display.font} ${r.display.px}px/${r.display.weight}` : '?'}\`${r.display ? ` (largest painted text: "${r.display.sample}")` : ''} · body \`${r.bodyFont}\`${r.fontFaces.length ? ` · @font-face: ${r.fontFaces.join(', ')}` : ''}`);377  lines.push(`- palette (most painted): ${r.palette.join(' · ')}`);378  if (r.shots.length) lines.push(`- shots: ${r.shots.map(s => `\`${s}\``).join(' ')}  ← look at the hero; look at the rest only if you take a steal from this one`);379  if (r.shotErrors?.length) lines.push(`- ⚠ ${r.shotErrors.length} of ${r.shotsWanted} shots failed (${r.shotErrors.join('; ')}) — the fingerprint above still holds.`);380  lines.push('- **steal:** _<one mechanic, one line — fill this in>_');381  lines.push('');382});383 384const failed = results.filter(r => r.error);385if (failed.length) {386  lines.push('## Not reached');387  failed.forEach(r => lines.push(`- ${r.url} — ${r.error}`));388  lines.push('');389}390 391await writeFile(join(outDir, 'REFERENCES.md'), lines.join('\n'), 'utf8');392await writeFile(join(outDir, 'refs.json'), JSON.stringify(results, null, 2), 'utf8');393 394console.log(`\n=== refscout ===`);395console.log(`profiled : ${ok.filter(r => !r.thin).length}/${results.length} usable`);396if (ok.some(r => r.thin)) console.log(`no-css   : ${ok.filter(r => r.thin).length} (site withheld CSS/JS from headless — reported as unknown, not guessed)`);397if (failed.length) console.log(`failed   : ${failed.length} — ${failed.map(r => r.url).join(', ')}`);398const lostShots = ok.reduce((n, r) => n + (r.shotErrors?.length || 0), 0);399if (lostShots) console.log(`shots    : ${lostShots} capture(s) failed — flagged per site in the report`);400console.log(`report   : ${join(outDir, 'REFERENCES.md')}`);401console.log(`shots    : ${ok.reduce((n, r) => n + r.shots.length, 0)} in ${join(outDir, 'shots')}`);402process.exit(ok.length ? 0 : 1);403