llm-torch-profiler-analysis

1#!/usr/bin/env bash2set -euo pipefail3 4usage() {5  cat <<'EOF'6Usage:7  run_sglang_torch_profile_host.sh \8    --model Qwen/Qwen3-8B \9    --run-dir /data/bbuf/validate/unified_llm_profiler_skill/runs/example_sglang \10    --port 30088 \11    --gpus 012 13  run_sglang_torch_profile_host.sh \14    --model openai/gpt-oss-20b \15    --run-dir /data/bbuf/validate/unified_llm_profiler_skill/runs/example_sglang_4gpu \16    --port 30088 \17    --gpus 2,3,4,5 \18    --tp-size 419 20Options:21  --model TEXT                  Model id or local path for SGLang.22  --run-dir PATH               Shared /data directory for logs and traces.23  --port INT                   Server port.24  --gpus TEXT                  CUDA_VISIBLE_DEVICES value, for example 0 or 2,3,4,5.25  --gpu TEXT                   Alias for --gpus.26  --tp-size INT                Tensor parallel size. Defaults to the visible GPU count.27  --trust-remote-code          Pass --trust-remote-code.28  --mem-fraction FLOAT         SGLang static memory fraction.29  --request-max-tokens INT     Generation length for the probe request.30  --prompt TEXT                Probe prompt.31  --warmup-steps INT           Warmup steps before profiling. Defaults to 10.32  --profile-workload TEXT      legacy|prefill|decode|both. Defaults to both.33  --prefill-input-len INT      Synthetic prefill prompt length. Defaults to 4090.34  --prefill-output-len INT     Synthetic prefill output length. Defaults to 1.35  --decode-input-len INT       Synthetic decode prompt length. Defaults to 1.36  --decode-output-len INT      Synthetic decode output length. Defaults to 2048.37  --repo-dir PATH              SGLang repo path inside `sglang_bbuf`.38  --server-extra TEXT          Extra args appended to launch_server.39  --help                       Show this message.40 41Notes:42  - Run this on the H100 host. It uses `docker exec sglang_bbuf`.43  - The server is launched first, then the profiler capture runs with44    stage-separated prefill/decode workloads and `--profile-by-stage`.45  - A small benchmark summary is written after profiling.46EOF47}48 49MODEL=""50RUN_DIR=""51PORT=""52GPUS=""53TP_SIZE=""54TRUST_REMOTE_CODE=055MEM_FRACTION=0.8556REQUEST_MAX_TOKENS=1257PROMPT="Explain the difference between CUDA graph mode and eager mode in two sentences."58WARMUP_STEPS=1059PROFILE_WORKLOAD="both"60PREFILL_INPUT_LEN=409061PREFILL_OUTPUT_LEN=162DECODE_INPUT_LEN=163DECODE_OUTPUT_LEN=204864SGLANG_REPO_DIR="${SGLANG_REPO_DIR:-/data/bbuf/repos/sglang}"65SERVER_EXTRA=""66SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"67 68while [[ $# -gt 0 ]]; do69  case "$1" in70    --model)71      MODEL="$2"72      shift 273      ;;74    --run-dir)75      RUN_DIR="$2"76      shift 277      ;;78    --port)79      PORT="$2"80      shift 281      ;;82    --gpu)83      GPUS="$2"84      shift 285      ;;86    --gpus)87      GPUS="$2"88      shift 289      ;;90    --tp-size)91      TP_SIZE="$2"92      shift 293      ;;94    --trust-remote-code)95      TRUST_REMOTE_CODE=196      shift97      ;;98    --mem-fraction)99      MEM_FRACTION="$2"100      shift 2101      ;;102    --request-max-tokens)103      REQUEST_MAX_TOKENS="$2"104      shift 2105      ;;106    --prompt)107      PROMPT="$2"108      shift 2109      ;;110    --warmup-steps)111      WARMUP_STEPS="$2"112      shift 2113      ;;114    --profile-workload)115      PROFILE_WORKLOAD="$2"116      shift 2117      ;;118    --prefill-input-len)119      PREFILL_INPUT_LEN="$2"120      shift 2121      ;;122    --prefill-output-len)123      PREFILL_OUTPUT_LEN="$2"124      shift 2125      ;;126    --decode-input-len)127      DECODE_INPUT_LEN="$2"128      shift 2129      ;;130    --decode-output-len)131      DECODE_OUTPUT_LEN="$2"132      shift 2133      ;;134    --repo-dir)135      SGLANG_REPO_DIR="$2"136      shift 2137      ;;138    --server-extra)139      SERVER_EXTRA="$2"140      shift 2141      ;;142    --help|-h)143      usage144      exit 0145      ;;146    *)147      echo "Unknown argument: $1" >&2148      usage >&2149      exit 2150      ;;151  esac152done153 154if [[ -z "$MODEL" || -z "$RUN_DIR" || -z "$PORT" || -z "$GPUS" ]]; then155  usage >&2156  exit 2157fi158 159IFS=',' read -r -a GPU_LIST <<< "$GPUS"160GPU_COUNT="${#GPU_LIST[@]}"161if [[ "$GPU_COUNT" -lt 1 ]]; then162  echo "Could not parse --gpus: $GPUS" >&2163  exit 2164fi165if [[ -z "$TP_SIZE" ]]; then166  TP_SIZE="$GPU_COUNT"167fi168if (( TP_SIZE < 1 || TP_SIZE > GPU_COUNT )); then169  echo "--tp-size must be between 1 and the visible GPU count ($GPU_COUNT)." >&2170  exit 2171fi172 173LOG_PATH="$RUN_DIR/sglang_server.log"174ANALYSIS_PATH="$RUN_DIR/analysis_sglang.txt"175PROFILE_ROOT="$RUN_DIR/sglang_profile_live"176BENCHMARK_PATH="$RUN_DIR/benchmark_sglang.json"177PID_PATH="$RUN_DIR/sglang_server.pid"178LAUNCH_PATTERN="[s]glang.launch_server.*--port $PORT"179SERVER_ARGS="python3 -m sglang.launch_server --model-path \"$MODEL\" --port \"$PORT\" --tp-size \"$TP_SIZE\" --mem-fraction-static \"$MEM_FRACTION\""180 181if [[ "$TRUST_REMOTE_CODE" -eq 1 ]]; then182  SERVER_ARGS="$SERVER_ARGS --trust-remote-code"183fi184if [[ -n "$SERVER_EXTRA" ]]; then185  SERVER_ARGS="$SERVER_ARGS $SERVER_EXTRA"186fi187 188docker exec sglang_bbuf bash -lc "mkdir -p '$RUN_DIR' '$PROFILE_ROOT'"189docker exec sglang_bbuf bash -lc "pkill -f '$LAUNCH_PATTERN' >/dev/null 2>&1 || true"190docker exec sglang_bbuf bash -lc "mkdir -p '$RUN_DIR' '$PROFILE_ROOT' && cd '$SGLANG_REPO_DIR' && rm -f '$PID_PATH' && (CUDA_VISIBLE_DEVICES=$GPUS PYTHONPATH=python nohup $SERVER_ARGS > '$LOG_PATH' 2>&1 < /dev/null & echo \$! > '$PID_PATH')"191 192cleanup() {193  docker exec sglang_bbuf bash -lc "pkill -f '$LAUNCH_PATTERN' >/dev/null 2>&1 || true" >/dev/null 2>&1 || true194}195trap cleanup EXIT196 197ready=0198for _ in $(seq 1 180); do199  if curl -sf "http://127.0.0.1:${PORT}/v1/models" >/dev/null; then200    ready=1201    break202  fi203  sleep 2204done205if [[ "$ready" -ne 1 ]]; then206  echo "SGLang server did not become ready on port ${PORT}. Recent logs:" >&2207  ssh_log=$(docker exec sglang_bbuf bash -lc "tail -n 120 '$LOG_PATH'" 2>/dev/null || true)208  printf '%s\n' "$ssh_log" >&2209  exit 1210fi211 212python3 - <<PY213import json214import urllib.request215 216payload = {217    "text": ${PROMPT@Q},218    "sampling_params": {219        "temperature": 0.0,220        "max_new_tokens": int(${REQUEST_MAX_TOKENS@Q}),221    },222    "stream": False,223}224req = urllib.request.Request(225    "http://127.0.0.1:${PORT}/generate",226    data=json.dumps(payload).encode(),227    headers={"Content-Type": "application/json"},228)229with urllib.request.urlopen(req, timeout=600) as resp:230    body = json.loads(resp.read().decode())231text = body.get("text", "")232print(text[:400])233PY234 235docker exec sglang_bbuf bash -lc "cd '$SCRIPT_DIR' && python3 analyze_llm_torch_profile.py --framework sglang --url http://127.0.0.1:${PORT} --output-dir '$PROFILE_ROOT' --num-steps 5 --warmup-steps '$WARMUP_STEPS' --probe-requests 1 --profile-by-stage --profile-workload '$PROFILE_WORKLOAD' --prefill-input-len '$PREFILL_INPUT_LEN' --prefill-output-len '$PREFILL_OUTPUT_LEN' --decode-input-len '$DECODE_INPUT_LEN' --decode-output-len '$DECODE_OUTPUT_LEN' > '$ANALYSIS_PATH'"236python3 "$SCRIPT_DIR/probe_llm_server.py" \237  --framework sglang \238  --url "http://127.0.0.1:${PORT}" \239  | docker exec -i sglang_bbuf bash -lc "cat > '$BENCHMARK_PATH'" >/dev/null240docker exec sglang_bbuf bash -lc "sed -n '1,240p' '$ANALYSIS_PATH'"241echo "BENCHMARK_PATH=$BENCHMARK_PATH"242