llm-torch-profiler-analysis

1"""Compact triage entrypoint for unified LLM torch-profiler analysis."""2 3from __future__ import annotations4 5import argparse6import sys7from collections import defaultdict8from pathlib import Path9from typing import Dict, List, Optional, Sequence, Tuple10 11import triage_kernel_helpers as kernel_helpers12import triage_overlap_helpers as overlap_helpers13from profile_common import (14    DEFAULT_DECODE_INPUT_LEN,15    DEFAULT_DECODE_OUTPUT_LEN,16    DEFAULT_PREFILL_INPUT_LEN,17    DEFAULT_PREFILL_OUTPUT_LEN,18    DEFAULT_WARMUP_STEPS,19    PROFILE_WORKLOAD_CHOICES,20    discover_trace_targets,21    framework_display_name,22    load_server_args,23    load_trace_json,24    parse_stage,25    resolve_framework,26    run_profiler,27)28 29MIN_RENDER_SHARE_PCT = 1.030MAPPING_KERNEL_SAMPLE_LIMIT_PER_NAME = 1631 32 33def build_triage_parser() -> argparse.ArgumentParser:34    parser = argparse.ArgumentParser(35        prog="analyze_llm_torch_profile.py",36        description=(37            "Compact LLM torch-profiler triage entrypoint for SGLang, vLLM, "38            "TensorRT-LLM, and TokenSpeed. "39            "This prints three tables: kernel mapping, overlap opportunities, "40            "and fuse opportunities. "41            "Use either a single trace/profile input or a mapping+formal two-trace pair."42        ),43    )44    parser.add_argument(45        "--framework",46        type=str,47        default="auto",48        choices=[49            "auto",50            "sglang",51            "vllm",52            "trtllm",53            "tllm",54            "tensorrt-llm",55            "tokenspeed",56            "token-speed",57            "ts",58        ],59        help=(60            "Serving framework. Use auto to detect from trace contents, path hints, "61            "or URL features."62        ),63    )64    parser.add_argument(65        "--input",66        type=str,67        default=None,68        help="Single trace file or profile directory to triage.",69    )70    parser.add_argument(71        "--url",72        type=str,73        default=None,74        help=(75            "Running server URL for single-trace triage. SGLang supports direct "76            "capture via sglang.profiler. vLLM and TensorRT-LLM require a server-side "77            "torch-profiler output path exposed via --output-dir. TokenSpeed live "78            "capture uses the server's /start_profile and /stop_profile endpoints "79            "when they are available."80        ),81    )82    parser.add_argument(83        "--output-dir",84        type=str,85        default=None,86        help=(87            "Trace output dir when using --url. For vLLM this should match the "88            "server's torch_profiler_dir. For TensorRT-LLM it should match the "89            "directory or file path configured by TLLM_TORCH_PROFILE_TRACE. "90            "For TokenSpeed this is passed as start_profile.output_dir."91        ),92    )93    parser.add_argument(94        "--profile-prefix",95        type=str,96        default="triage-trace",97        help=(98            "Profile prefix when generating a trace from --url. SGLang uses it "99            "directly; TokenSpeed maps it to profile_id; vLLM and TensorRT-LLM may "100            "ignore it on the HTTP profiler path."101        ),102    )103    parser.add_argument(104        "--mapping-input",105        type=str,106        default=None,107        help="Graph-off mapping trace file or directory.",108    )109    parser.add_argument(110        "--mapping-url",111        type=str,112        default=None,113        help="Running graph-off server URL for the mapping trace.",114    )115    parser.add_argument(116        "--formal-input",117        type=str,118        default=None,119        help="Formal graph-on trace file or directory.",120    )121    parser.add_argument(122        "--formal-url",123        type=str,124        default=None,125        help="Running graph-on server URL for the formal trace.",126    )127    parser.add_argument(128        "--mapping-output-dir",129        type=str,130        default=None,131        help="Trace output dir when using --mapping-url.",132    )133    parser.add_argument(134        "--formal-output-dir",135        type=str,136        default=None,137        help="Trace output dir when using --formal-url.",138    )139    parser.add_argument(140        "--mapping-profile-prefix",141        type=str,142        default="mapping-trace",143        help="Profile prefix for the mapping trace.",144    )145    parser.add_argument(146        "--formal-profile-prefix",147        type=str,148        default="formal-trace",149        help="Profile prefix for the formal trace.",150    )151    parser.add_argument(152        "--num-steps",153        type=int,154        default=5,155        help="Active profiler steps when generating traces from URLs.",156    )157    parser.add_argument(158        "--warmup-steps",159        type=int,160        default=DEFAULT_WARMUP_STEPS,161        help="Warmup steps to run before arming the profiler for URL capture.",162    )163    parser.add_argument(164        "--profile-by-stage", action=argparse.BooleanOptionalAction, default=True165    )166    parser.add_argument(167        "--merge-profiles", action=argparse.BooleanOptionalAction, default=False168    )169    parser.add_argument("--probe-requests", type=int, default=1)170    parser.add_argument(171        "--probe-prompt",172        type=str,173        default=(174            "Repeat the word profiler many times with spaces so the server performs several decode steps. "175            "Do not add explanations."176        ),177    )178    parser.add_argument("--probe-max-new-tokens", type=int, default=None)179    parser.add_argument("--probe-delay", type=float, default=0.5)180    parser.add_argument(181        "--profile-workload",182        choices=PROFILE_WORKLOAD_CHOICES,183        default="both",184        help=(185            "Live-capture workload shape. Default 'both' captures separate "186            "prefill and decode profiles instead of one mixed request. Use "187            "'legacy' to keep the old --probe-prompt behavior."188        ),189    )190    parser.add_argument(191        "--prefill-input-len",192        type=int,193        default=DEFAULT_PREFILL_INPUT_LEN,194        help="Synthetic input length for the prefill profile workload.",195    )196    parser.add_argument(197        "--prefill-output-len",198        type=int,199        default=DEFAULT_PREFILL_OUTPUT_LEN,200        help="Output length for the prefill profile workload.",201    )202    parser.add_argument(203        "--decode-input-len",204        type=int,205        default=DEFAULT_DECODE_INPUT_LEN,206        help="Synthetic input length for the decode profile workload.",207    )208    parser.add_argument(209        "--decode-output-len",210        type=int,211        default=DEFAULT_DECODE_OUTPUT_LEN,212        help="Output length for the decode profile workload.",213    )214    parser.add_argument(215        "--start-step",216        type=int,217        default=None,218        help="Pass through to sglang.profiler when generating traces from URLs.",219    )220    parser.add_argument(221        "--pid-substring",222        type=str,223        default=None,224        help="Restrict overlap analysis to PIDs containing this substring.",225    )226    parser.add_argument(227        "--kernel-table-limit",228        type=int,229        default=0,230        help="How many kernel rows to print per stage. Use 0 for all kernels.",231    )232    parser.add_argument(233        "--overlap-table-limit",234        type=int,235        default=0,236        help="How many overlap rows to print per stage. Use 0 for all kernels.",237    )238    return parser239 240 241def parse_triage_args(argv: Sequence[str]) -> argparse.Namespace:242    parser = build_triage_parser()243    args = parser.parse_args(argv)244 245    single_trace_mode = bool(args.input) or bool(args.url)246    dual_trace_mode = any(247        [248            args.mapping_input,249            args.mapping_url,250            args.formal_input,251            args.formal_url,252        ]253    )254 255    if single_trace_mode and dual_trace_mode:256        parser.error(257            "Use either single-trace mode (--input/--url) or two-trace mode "258            "(--mapping-* plus --formal-*), not both."259        )260 261    if single_trace_mode:262        if bool(args.input) == bool(args.url):263            parser.error("Provide exactly one of --input or --url.")264        return args265 266    if bool(args.mapping_input) == bool(args.mapping_url):267        parser.error("Provide exactly one of --mapping-input or --mapping-url.")268    if bool(args.formal_input) == bool(args.formal_url):269        parser.error("Provide exactly one of --formal-input or --formal-url.")270    return args271 272 273def resolve_profile_targets(274    *,275    label: str,276    input_path: Optional[str],277    url: Optional[str],278    output_dir: Optional[str],279    profile_prefix: Optional[str],280    args: argparse.Namespace,281) -> Tuple[List[Path], Optional[dict], str]:282    if bool(input_path) == bool(url):283        raise ValueError(f"{label} trace requires exactly one of input path or URL.")284 285    if url:286        framework = resolve_framework(287            args.framework,288            input_path=Path(output_dir).resolve() if output_dir else None,289            url=url,290        )291        target_dir = run_profiler(292            url=url,293            output_dir=output_dir,294            num_steps=args.num_steps,295            profile_by_stage=args.profile_by_stage,296            merge_profiles=args.merge_profiles,297            profile_prefix=profile_prefix,298            probe_requests=max(0, args.probe_requests),299            probe_prompt=args.probe_prompt,300            probe_max_new_tokens=args.probe_max_new_tokens,301            probe_delay=args.probe_delay,302            warmup_steps=args.warmup_steps,303            start_step=args.start_step,304            framework=framework,305            framework_hint_path=output_dir,306            profile_workload=args.profile_workload,307            prefill_input_len=args.prefill_input_len,308            prefill_output_len=args.prefill_output_len,309            decode_input_len=args.decode_input_len,310            decode_output_len=args.decode_output_len,311        )312        traces, server_args = discover_trace_targets(target_dir, all_traces=False)313        resolved_framework = resolve_framework(314            args.framework,315            input_path=target_dir,316            url=url,317            server_args=server_args,318        )319        return traces, server_args, resolved_framework320 321    resolved = Path(input_path).resolve()322    traces, server_args = discover_trace_targets(resolved, all_traces=False)323    if server_args is None:324        server_args = load_server_args(resolved)325    framework = resolve_framework(326        args.framework, input_path=resolved, server_args=server_args327    )328    return traces, server_args, framework329 330 331def build_mapping_kernel_map(trace_paths: Sequence[Path], framework: str) -> dict:332    stage_site_stats = defaultdict(333        lambda: defaultdict(lambda: defaultdict(kernel_helpers.MappingSiteAggregate))334    )335    stage_kernel_categories: Dict[str, Dict[str, str]] = defaultdict(dict)336    global_site_stats = defaultdict(337        lambda: defaultdict(kernel_helpers.MappingSiteAggregate)338    )339    global_kernel_categories: Dict[str, str] = {}340 341    for trace_path in trace_paths:342        trace = load_trace_json(trace_path)343        kernels, cpu_ops, python_frames, launch_events, _, _ = (344            kernel_helpers.extract_trace_data(trace)345        )346        if not kernels:347            continue348        cpu_ops_by_external_id = kernel_helpers.build_cpu_op_index(cpu_ops)349        launches_by_correlation = kernel_helpers.build_launch_index(launch_events)350        site_context_cache = {}351        default_stage = parse_stage(trace_path)352        for stage, stage_kernels in kernel_helpers.group_kernels_by_stage(353            kernels, default_stage354        ).items():355            sampled_stage_kernels = (356                stage_kernels357                if framework == "sglang"358                else sample_kernels_for_mapping(stage_kernels)359            )360            local_site_stats = kernel_helpers.aggregate_kernel_sites(361                sampled_stage_kernels,362                cpu_ops_by_external_id,363                python_frames,364                launches_by_correlation=launches_by_correlation,365                site_context_cache=site_context_cache,366            )367            kernel_categories = {368                kernel.canonical_name: kernel.category for kernel in stage_kernels369            }370            kernel_helpers.merge_site_stats(stage_site_stats[stage], local_site_stats)371            kernel_helpers.merge_site_stats(global_site_stats, local_site_stats)372            stage_kernel_categories[stage].update(kernel_categories)373            global_kernel_categories.update(kernel_categories)374 375    stage_payloads = {376        stage: kernel_helpers.build_stage_payload(377            dict(site_stats), stage_kernel_categories.get(stage, {})378        )379        for stage, site_stats in stage_site_stats.items()380    }381    global_payload = kernel_helpers.build_stage_payload(382        dict(global_site_stats), global_kernel_categories383    )384    return {"stages": stage_payloads, "global": global_payload}385 386 387def stage_index(stage: str) -> int:388    return {"extend": 0, "prefill": 0, "decode": 1, "all": 2}.get(stage, 99)389 390 391def sample_kernels_for_mapping(392    kernels: Sequence[kernel_helpers.KernelEvent],393    per_name_limit: int = MAPPING_KERNEL_SAMPLE_LIMIT_PER_NAME,394) -> List[kernel_helpers.KernelEvent]:395    if per_name_limit <= 0:396        return list(kernels)397 398    grouped: Dict[str, List[kernel_helpers.KernelEvent]] = defaultdict(list)399    for kernel in kernels:400        grouped[kernel.canonical_name].append(kernel)401 402    sampled: List[kernel_helpers.KernelEvent] = []403    for kernel_name in sorted(grouped):404        items = grouped[kernel_name]405        if len(items) <= per_name_limit:406            sampled.extend(items)407            continue408        for sample_idx in range(per_name_limit):409            pos = round(sample_idx * (len(items) - 1) / (per_name_limit - 1))410            sampled.append(items[pos])411    sampled.sort(key=lambda kernel: (kernel.ts, kernel.name))412    return sampled413 414 415def stage_display(stage: str) -> str:416    return kernel_helpers.stage_label(stage)417 418 419def pick_stage_value(stage_to_value: Dict[str, object], stage: str) -> Optional[object]:420    if stage in stage_to_value:421        return stage_to_value[stage]422    if "all" in stage_to_value:423        return stage_to_value["all"]424    if len(stage_to_value) == 1:425        return next(iter(stage_to_value.values()))426    return None427 428 429def render_stages(stage_to_value: Dict[str, object]) -> List[str]:430    stages = set(stage_to_value)431    if any(stage != "all" for stage in stages):432        stages.discard("all")433    return sorted(stages, key=stage_index)434 435 436def build_overlap_stage_bundle_map(437    trace_paths: Sequence[Path],438    *,439    label_prefix: str,440    server_args: Optional[dict],441    pid_substring: Optional[str],442) -> Dict[str, overlap_helpers.TraceBundle]:443    stage_bundles: Dict[str, overlap_helpers.TraceBundle] = {}444    for trace_path in sorted(445        trace_paths, key=lambda item: (stage_index(parse_stage(item)), item.name)446    ):447        trace_json = load_trace_json(trace_path)448        raw_events = trace_json.get(449            "traceEvents",450            trace_json if isinstance(trace_json, list) else [],451        )452        events, pid = overlap_helpers.extract_kernel_events(trace_json, pid_substring)453        if not events:454            continue455        default_stage = parse_stage(trace_path)456        stage_groups = overlap_helpers.group_events_by_stage(events, default_stage)457        for stage in render_stages(stage_groups):458            if stage in stage_bundles:459                continue460            stage_bundles[stage] = overlap_helpers.TraceBundle(461                label=f"{label_prefix}-{stage}",462                trace_path=trace_path,463                server_args=server_args,464                raw_events=raw_events,465                events=stage_groups[stage],466                pid=pid,467            )468        if "all" in stage_groups and not stage_bundles:469            stage_bundles["all"] = overlap_helpers.TraceBundle(470                label=f"{label_prefix}-all",471                trace_path=trace_path,472                server_args=server_args,473                raw_events=raw_events,474                events=stage_groups["all"],475                pid=pid,476            )477    return stage_bundles478 479 480def group_rows_by_stage(rows: Sequence[dict]) -> List[Tuple[str, List[dict]]]:481    grouped: Dict[str, List[dict]] = defaultdict(list)482    for row in rows:483        grouped[str(row.get("stage") or "all")].append(row)484    return [485        (stage, grouped[stage]) for stage in sorted(grouped.keys(), key=stage_index)486    ]487 488 489def render_kernel_table_for_stage(rows: Sequence[dict]) -> List[str]:490    lines = [491        "| Kernel | Category | GPU time | Share | Launches | Python location (site share) | CPU op |",492        "| --- | --- | ---: | ---: | ---: | --- | --- |",493    ]494    if not rows:495        lines.append(496            "| No kernel rows at or above 1.0% share. | - | - | - | - | - | - |"497        )498        return lines499    for row in rows:500        lines.append(501            "| {kernel} | {category} | {gpu_time} | {share:.1f}% | {launches} | {location} | {cpu_op} |".format(502                kernel=kernel_helpers.escape_md_cell(row["kernel"]),503                category=kernel_helpers.escape_md_cell(row["category"]),504                gpu_time=kernel_helpers.format_ms(row["total_us"]),505                share=row["share_pct"],506                launches=row["launches"],507                location=kernel_helpers.escape_md_cell(row["location"]),508                cpu_op=kernel_helpers.escape_md_cell(row["cpu_op"]),509            )510        )511    return lines512 513 514def render_stage_section_tables(515    rows: Sequence[dict],516    *,517    render_stage_fn,518    stage_label_prefix: str = "#####",519) -> List[str]:520    if not rows:521        return render_stage_fn([])522    stage_groups = group_rows_by_stage(rows)523    if len(stage_groups) == 1 and stage_groups[0][0] == "all":524        return render_stage_fn(stage_groups[0][1])525 526    lines: List[str] = []527    for index, (stage, stage_rows) in enumerate(stage_groups):528        lines.append(f"{stage_label_prefix} {stage_display(stage)}")529        lines.extend(render_stage_fn(stage_rows))530        if index != len(stage_groups) - 1:531            lines.append("")532    return lines533 534 535def render_kernel_tables(rows: Sequence[dict]) -> List[str]:536    return render_stage_section_tables(537        rows, render_stage_fn=render_kernel_table_for_stage538    )539 540 541def render_overlap_table_for_stage(rows: Sequence[dict]) -> List[str]:542    lines = [543        "| Priority | Verdict | Kernel | Python scope | Formal signal | Dep risk | Recommendation |",544        "| --- | --- | --- | --- | --- | --- | --- |",545    ]546    if not rows:547        lines.append(548            "| - | - | No rows cleared the 1.0% reporting bar. Use mapping/formal mode for overlap attribution. | - | - | - | - |"549        )550        return lines551    for row in rows:552        formal_signal = (553            f"{row['total_us']:.1f} us, share {row['share_pct']:.1f}%, "554            f"excl {row['exclusive_ratio'] * 100:.1f}% / hid {row['hidden_ratio'] * 100:.1f}%"555        )556        lines.append(557            "| "558            + " | ".join(559                [560                    row["priority"],561                    row["verdict"],562                    kernel_helpers.escape_md_cell(row["kernel"]),563                    kernel_helpers.escape_md_cell(row["python_scope"]),564                    kernel_helpers.escape_md_cell(formal_signal),565                    overlap_helpers.dependency_risk_label(row["dependency_signal"]),566                    row["recommendation"],567                ]568            )569            + " |"570        )571    return lines572 573 574def render_overlap_tables(rows: Sequence[dict]) -> List[str]:575    return render_stage_section_tables(576        rows,577        render_stage_fn=render_overlap_table_for_stage,578    )579 580 581def render_fuse_table_for_stage(rows: Sequence[dict]) -> List[str]:582    lines = [583        "| Pattern | Confidence | Related GPU time | Share | Evidence kernels | Current kernel Python location | Candidate fused Python path | Rationale |",584        "| --- | --- | ---: | ---: | --- | --- | --- | --- |",585    ]586    if not rows:587        lines.append(588            "| No medium-confidence source-backed fusion opportunity matched this trace. | - | - | - | - | - | - | - |"589        )590        return lines591    for row in rows:592        lines.append(593            "| {pattern} | {confidence} | {gpu_time} | {share:.1f}% | {evidence} | {current_locations} | {candidate_path} | {rationale} |".format(594                pattern=kernel_helpers.escape_md_cell(row["pattern"]),595                confidence=kernel_helpers.escape_md_cell(row["confidence"]),596                gpu_time=kernel_helpers.format_ms(row["related_us"]),597                share=row["share_pct"],598                evidence=kernel_helpers.escape_md_cell(row["evidence"]),599                current_locations=kernel_helpers.escape_md_cell(600                    row["current_locations"]601                ),602                candidate_path=kernel_helpers.escape_md_cell(row["candidate_path"]),603                rationale=kernel_helpers.escape_md_cell(row["rationale"]),604            )605        )606    return lines607 608 609def render_fuse_tables(rows: Sequence[dict]) -> List[str]:610    return render_stage_section_tables(611        rows,612        render_stage_fn=render_fuse_table_for_stage,613    )614 615 616def run_triage(args: argparse.Namespace) -> int:617    single_trace_mode = bool(args.input) or bool(args.url)618    if single_trace_mode:619        formal_traces, formal_server_args, formal_framework = resolve_profile_targets(620            label="input",621            input_path=args.input,622            url=args.url,623            output_dir=args.output_dir,624            profile_prefix=args.profile_prefix,625            args=args,626        )627        mapping_traces = formal_traces628        mapping_server_args = formal_server_args629        mapping_framework = formal_framework630    else:631        mapping_traces, mapping_server_args, mapping_framework = (632            resolve_profile_targets(633                label="mapping",634                input_path=args.mapping_input,635                url=args.mapping_url,636                output_dir=args.mapping_output_dir,637                profile_prefix=args.mapping_profile_prefix,638                args=args,639            )640        )641        formal_traces, formal_server_args, formal_framework = resolve_profile_targets(642            label="formal",643            input_path=args.formal_input,644            url=args.formal_url,645            output_dir=args.formal_output_dir,646            profile_prefix=args.formal_profile_prefix,647            args=args,648        )649 650    mapping_kernel_map = build_mapping_kernel_map(mapping_traces, mapping_framework)651 652    kernel_rows_rendered: List[dict] = []653    fuse_rows_rendered: List[dict] = []654    formal_stage_payloads: Dict[str, dict] = {}655 656    for formal_trace in formal_traces:657        trace = load_trace_json(formal_trace)658        kernels, cpu_ops, python_frames, launch_events, _, _ = (659            kernel_helpers.extract_trace_data(trace)660        )661        if not kernels:662            continue663        default_stage = parse_stage(formal_trace)664        stage_groups = kernel_helpers.group_kernels_by_stage(kernels, default_stage)665        formal_cpu_ops_by_external_id = kernel_helpers.build_cpu_op_index(cpu_ops)666        formal_launches_by_correlation = kernel_helpers.build_launch_index(667            launch_events668        )669        formal_site_context_cache = {}670        for stage_name, stage_kernels in stage_groups.items():671            local_site_stats = kernel_helpers.aggregate_kernel_sites(672                stage_kernels,673                formal_cpu_ops_by_external_id,674                python_frames,675                launches_by_correlation=formal_launches_by_correlation,676                site_context_cache=formal_site_context_cache,677            )678            formal_stage_payloads[stage_name] = kernel_helpers.build_stage_payload(679                local_site_stats,680                {kernel.canonical_name: kernel.category for kernel in stage_kernels},681            )682        trace_total_us = sum(kernel.dur for kernel in kernels)683        for stage in sorted(stage_groups, key=stage_index):684            stage_kernels = stage_groups[stage]685            if not stage_kernels:686                continue687            total_us = sum(kernel.dur for kernel in stage_kernels)688            if (689                stage == "all"690                and default_stage == "all"691                and kernel_helpers.pct(total_us, trace_total_us) < MIN_RENDER_SHARE_PCT692            ):693                continue694            kernel_stats = kernel_helpers.aggregate(695                stage_kernels, key_fn=lambda item: item.canonical_name696            )697            kernel_categories = {698                kernel.canonical_name: kernel.category for kernel in stage_kernels699            }700            full_kernel_rows = kernel_helpers.build_kernel_rows(701                stage=stage,702                kernel_stats=kernel_stats,703                kernel_categories=kernel_categories,704                local_stage_payload=formal_stage_payloads.get(stage, {"kernels": {}}),705                external_kernel_map=mapping_kernel_map,706            )707            visible_kernel_rows = kernel_helpers.limit_kernel_rows(708                full_kernel_rows, args.kernel_table_limit709            )710            for row in visible_kernel_rows:711                share_pct = kernel_helpers.pct(row.total_us, total_us)712                if share_pct < MIN_RENDER_SHARE_PCT:713                    continue714                kernel_rows_rendered.append(715                    {716                        "stage": stage,717                        "kernel": row.name,718                        "category": row.category,719                        "total_us": row.total_us,720                        "share_pct": share_pct,721                        "launches": row.aggregate.count,722                        "location": row.location,723                        "cpu_op": row.cpu_op,724                    }725                )726            for item in kernel_helpers.detect_fusion_opportunities(727                kernel_rows=full_kernel_rows,728                total_us=total_us,729                server_args=formal_server_args or mapping_server_args,730                framework=formal_framework,731            ):732                share_pct = kernel_helpers.pct(item.related_us, total_us)733                if share_pct < MIN_RENDER_SHARE_PCT:734                    continue735                fuse_rows_rendered.append(736                    {737                        "stage": stage,738                        "pattern": item.pattern,739                        "confidence": item.confidence,740                        "related_us": item.related_us,741                        "share_pct": share_pct,742                        "evidence": item.evidence,743                        "current_locations": item.current_locations,744                        "candidate_path": item.candidate_path,745                        "rationale": item.rationale,746                    }747                )748 749    overlap_rows_rendered: List[dict] = []750    if not single_trace_mode:751        mapping_overlap_bundles = build_overlap_stage_bundle_map(752            mapping_traces,753            label_prefix="mapping",754            server_args=mapping_server_args,755            pid_substring=args.pid_substring,756        )757        formal_overlap_bundles = build_overlap_stage_bundle_map(758            formal_traces,759            label_prefix="formal",760            server_args=formal_server_args,761            pid_substring=args.pid_substring,762        )763        for stage in render_stages(formal_overlap_bundles):764            formal_bundle = pick_stage_value(formal_overlap_bundles, stage)765            mapping_bundle = pick_stage_value(mapping_overlap_bundles, stage)766            if formal_bundle is None or mapping_bundle is None:767                continue768            formal_bundle.overlap_stats = overlap_helpers.analyze_overlap(769                formal_bundle.events770            )771            aggregates = overlap_helpers.aggregate_events(formal_bundle.events)772            source_map = overlap_helpers.build_kernel_source_map(773                mapping_bundle,774                kernel_map_entry_lookup=lambda stage_name, kernel_name: (775                    kernel_helpers.lookup_kernel_map_entry(776                        mapping_kernel_map, stage_name, kernel_name777                    )778                    if mapping_kernel_map779                    else None780                ),781                stage=stage,782            )783            source_map = overlap_helpers.merge_source_map_from_kernel_payload(784                source_map,785                pick_stage_value(formal_stage_payloads, stage),786            )787            stage_rows = overlap_helpers.build_action_rows(788                aggregates,789                source_map,790                formal_bundle.events,791                formal_bundle.overlap_stats["total_busy_us"],792                table_limit=max(0, args.overlap_table_limit),793            )794            for row in stage_rows:795                if row.share_pct < MIN_RENDER_SHARE_PCT:796                    continue797                overlap_rows_rendered.append(798                    {799                        "stage": stage,800                        "priority": row.priority,801                        "verdict": row.verdict,802                        "kernel": row.kernel,803                        "python_scope": row.python_scope,804                        "total_us": row.total_us,805                        "share_pct": row.share_pct,806                        "exclusive_ratio": row.exclusive_ratio,807                        "hidden_ratio": row.hidden_ratio,808                        "dependency_signal": row.dependency_signal,809                        "recommendation": row.recommendation,810                    }811                )812 813    lines: List[str] = []814    lines.append("Triage View")815    lines.append(f"Mode: {'single-trace' if single_trace_mode else 'mapping-formal'}")816    if single_trace_mode:817        lines.append(f"Framework: {framework_display_name(formal_framework)}")818        lines.append(f"Input traces: {', '.join(str(path) for path in formal_traces)}")819    else:820        if mapping_framework == formal_framework:821            lines.append(f"Framework: {framework_display_name(formal_framework)}")822        else:823            lines.append(824                f"Mapping framework: {framework_display_name(mapping_framework)}"825            )826            lines.append(827                f"Formal framework: {framework_display_name(formal_framework)}"828            )829        lines.append(830            f"Mapping traces: {', '.join(str(path) for path in mapping_traces)}"831        )832        lines.append(f"Formal traces: {', '.join(str(path) for path in formal_traces)}")833    if formal_server_args or mapping_server_args:834        server_args = formal_server_args or mapping_server_args835        model = server_args.get("model_path") or server_args.get("model")836        if model:837            lines.append(f"Model: {model}")838    lines.append("")839    lines.append("Kernel Table")840    lines.extend(render_kernel_tables(kernel_rows_rendered))841    lines.append("")842    lines.append("Overlap Opportunity Table")843    lines.extend(render_overlap_tables(overlap_rows_rendered))844    lines.append("")845    lines.append("Fuse Opportunity Table")846    lines.extend(render_fuse_tables(fuse_rows_rendered))847    print("\n".join(lines).rstrip())848    return 0849 850 851def main(argv: Optional[Sequence[str]] = None) -> int:852    argv = list(argv or sys.argv[1:])853    triage_parser = build_triage_parser()854 855    if not argv or argv[0] in {"-h", "--help"}:856        triage_parser.print_help()857        return 0858 859    if argv[0] == "triage":860        argv = argv[1:]861    elif not argv[0].startswith("-"):862        triage_parser.error(863            "This skill exposes only the triage workflow. "864            "Use single-trace mode (--input/--url) or mapping+formal two-trace mode."865        )866        return 2867 868    return run_triage(parse_triage_args(argv))869 870 871if __name__ == "__main__":872    raise SystemExit(main(sys.argv[1:]))873