osint-investigation

1"""Tiny stdlib HTTP helper used by fetch_*.py scripts.2 3Provides polite retry + JSON convenience + User-Agent enforcement.4"""5from __future__ import annotations6 7import json8import os9import time10import urllib.error11import urllib.parse12import urllib.request13 14DEFAULT_UA = (15    "hermes-osint-investigation/0.2 "16    "(+https://github.com/NousResearch/hermes-agent; "17    "set HERMES_OSINT_UA env var to identify yourself per "18    "Wikimedia / SEC fair-use guidance)"19)20 21 22def get(23    url: str,24    *,25    params: dict | None = None,26    headers: dict | None = None,27    user_agent: str | None = None,28    max_retries: int = 3,29    backoff: float = 1.5,30    timeout: float = 30.0,31) -> bytes:32    """GET with retry on 5xx and Retry-After honoring.33 34    429 (rate-limit) is raised IMMEDIATELY with a clear message — retrying35    when the upstream says "you're over quota" just wastes time. The caller36    should slow down or supply real credentials.37    """38    if params:39        sep = "&" if "?" in url else "?"40        url = f"{url}{sep}{urllib.parse.urlencode(params)}"41    h = {"User-Agent": user_agent or os.environ.get("HERMES_OSINT_UA", DEFAULT_UA)}42    if headers:43        h.update(headers)44 45    last_err: Exception | None = None46    for attempt in range(max_retries + 1):47        req = urllib.request.Request(url, headers=h)48        try:49            with urllib.request.urlopen(req, timeout=timeout) as resp:50                return resp.read()51        except urllib.error.HTTPError as e:52            if e.code == 429:53                # Surface immediately. Read the body so the caller sees the54                # provider's actual message ("OVER_RATE_LIMIT" etc.).55                try:56                    body = e.read(2048).decode("utf-8", errors="replace")57                except Exception:  # noqa: BLE00158                    body = ""59                raise RuntimeError(60                    f"HTTP 429 rate-limited by {urllib.parse.urlsplit(url).netloc}. "61                    f"Slow down or supply a real API key. Body: {body[:300]}"62                ) from e63            if e.code in {500, 502, 503, 504} and attempt < max_retries:64                retry_after = e.headers.get("Retry-After") if e.headers else None65                wait = float(retry_after) if (retry_after and retry_after.isdigit()) else backoff ** (attempt + 1)66                time.sleep(wait)67                last_err = e68                continue69            raise70        except urllib.error.URLError as e:71            if attempt < max_retries:72                time.sleep(backoff ** (attempt + 1))73                last_err = e74                continue75            raise76    if last_err:77        raise last_err78    raise RuntimeError("unreachable")79 80 81def get_json(url: str, **kwargs) -> dict | list:82    return json.loads(get(url, **kwargs).decode("utf-8"))83