har-derived-api-client

1#!/usr/bin/env python32"""Distill a HAR file into an API summary an agent can turn into a client.3 4Usage:5  python3 har_to_client.py <input.har> [--include-static] [--host SUBSTRING] [--max-body 600]6 7Filters to XHR/fetch/JSON traffic by default, groups by (method, host, path8template), and prints per-endpoint: query params, interesting request headers,9request body sample, response content-type/status, and a response body sample.10Numeric/UUID-ish path segments are collapsed to {id} so repeated calls group.11Also prints "### Replay hints": the browser User-Agent plus whether cookies or12auth/token headers were present -- send those in the derived client or you may13get a 403/401.14"""15import argparse16import json17import re18import sys19from collections import OrderedDict20from urllib.parse import urlsplit21 22BORING_HEADERS = {23    "accept-encoding", "accept-language", "connection", "content-length",24    "host", "origin", "referer", "sec-ch-ua", "sec-ch-ua-mobile",25    "sec-ch-ua-platform", "sec-fetch-dest", "sec-fetch-mode", "sec-fetch-site",26    "user-agent", "pragma", "cache-control", "priority", "te",27    "upgrade-insecure-requests", "cookie",28}29ID_SEG = re.compile(r"^(\d+|[0-9a-f]{8}-[0-9a-f-]{27,}|[0-9a-f]{16,})$", re.I)30STATIC_EXT = re.compile(r"\.(js|css|png|jpe?g|gif|svg|webp|ico|woff2?|ttf|mp4|map)$", re.I)31 32 33def path_template(path: str) -> str:34    segs = path.split("/")35    return "/".join("{id}" if ID_SEG.match(s) else s for s in segs)36 37 38def is_api_entry(entry: dict) -> bool:39    req = entry["request"]40    resp = entry.get("response", {})41    rtype = (entry.get("_resourceType") or "").lower()42    mime = (resp.get("content", {}).get("mimeType") or "").lower()43    if rtype in ("xhr", "fetch"):44        return True45    if "json" in mime:46        return True47    if req["method"] not in ("GET", "HEAD") and not STATIC_EXT.search(urlsplit(req["url"]).path):48        return True49    return False50 51 52def trunc(text, n: int) -> str:53    text = text if isinstance(text, str) else str(text)54    return text if len(text) <= n else text[:n] + f"... [{len(text)} chars total]"55 56 57def main() -> int:58    ap = argparse.ArgumentParser()59    ap.add_argument("har")60    ap.add_argument("--include-static", action="store_true")61    ap.add_argument("--host", default=None, help="only endpoints whose host contains this")62    ap.add_argument("--max-body", type=int, default=600)63    args = ap.parse_args()64 65    with open(args.har, encoding="utf-8") as f:66        har = json.load(f)67 68    groups = OrderedDict()69    for entry in har["log"]["entries"]:70        req = entry["request"]71        url = urlsplit(req["url"])72        if url.scheme not in ("http", "https"):73            continue74        if args.host and args.host not in url.netloc:75            continue76        if not args.include_static:77            if STATIC_EXT.search(url.path) or not is_api_entry(entry):78                continue79        key = (req["method"], url.netloc, path_template(url.path))80        g = groups.setdefault(key, {"count": 0, "queries": set(), "headers": {},81                                    "req_body": None, "resp": None})82        g["count"] += 183        for q in req.get("queryString", []):84            g["queries"].add((q["name"], trunc(q["value"], 80)))85        for h in req.get("headers", []):86            name = h["name"].lower().lstrip(":")87            if name in BORING_HEADERS or name in ("method", "path", "scheme", "authority"):88                continue89            g["headers"][name] = trunc(h["value"], 120)90        post = req.get("postData", {})91        if post.get("text") and g["req_body"] is None:92            g["req_body"] = (post.get("mimeType", ""), trunc(post["text"], args.max_body))93        resp = entry.get("response", {})94        if g["resp"] is None and resp:95            content = resp.get("content", {})96            g["resp"] = (resp.get("status"), content.get("mimeType", ""),97                         trunc(content.get("text") or "", args.max_body))98 99    if not groups:100        print("No API-looking entries found. Re-run with --include-static to see everything.")101        return 1102 103    # Surface the browser identity so the replay client can match it (many104    # sites 403 a default library User-Agent).105    ua = None106    saw_cookie = saw_auth = False107    for entry in har["log"]["entries"]:108        for h in entry["request"].get("headers", []):109            n = h["name"].lower()110            if n == "user-agent" and ua is None:111                ua = h["value"]112            if n == "cookie":113                saw_cookie = True114            if n in ("authorization", "x-api-key") or "token" in n:115                saw_auth = True116    print("### Replay hints")117    if ua:118        print(f"  User-Agent (send this): {ua}")119    if saw_cookie:120        print("  Cookies present -> session may be auth-gated; capture & resend the Cookie header.")121    if saw_auth:122        print("  Authorization/token header present -> extract and resend it.")123 124    for (method, host, path), g in groups.items():125        print(f"\n=== {method} https://{host}{path}  (x{g['count']})")126        if g["queries"]:127            print("  query params:")128            for name, val in sorted(g["queries"]):129                print(f"    {name} = {val}")130        if g["headers"]:131            print("  request headers (non-boring):")132            for name, val in sorted(g["headers"].items()):133                print(f"    {name}: {val}")134        if g["req_body"]:135            print(f"  request body ({g['req_body'][0]}):")136            print(f"    {g['req_body'][1]}")137        if g["resp"]:138            status, mime, body = g["resp"]139            print(f"  response: {status} {mime}")140            if body:141                print(f"    {body}")142    print(f"\n{len(groups)} distinct endpoints.")143    return 0144 145 146if __name__ == "__main__":147    sys.exit(main())148