cookbook-add-model

1// Diffusion command-builder config template. Replace every __TOKEN__, keep only2// verified hardware/features, and adapt the topology divisibility contract to3// the model architecture.4 5export const config = {6  modelName: "__MODEL_DISPLAY__",7  supportedHardware: ["__DEFAULT_HW__"],8  groupHardware: false,9  matchDims: [],10 11  overlayDims: [12    {13      id: "weights",14      title: "Checkpoint weights",15      scope: "base",16      description: "Choose the checkpoint partition required by this request mode.",17      default: "default",18      options: [{ id: "default", label: "Default", flags: [] }],19    },20    {21      id: "mode",22      title: "Request mode",23      scope: "base",24      default: "text",25      options: [{ id: "text", label: "Text" }],26    },27    {28      id: "placement",29      title: "Placement",30      scope: "serve",31      description: "Keep weights resident unless a verified capacity path requires sharding or offload.",32      default: "resident",33      options: [34        { id: "resident", label: "Resident", flags: ["--performance-mode speed"], recommended: true },35        { id: "fsdp", label: "FSDP", flags: ["--use-fsdp-inference true"], disabled: true },36        { id: "offload", label: "Layerwise offload", disabled: true },37      ],38    },39    {40      id: "attention",41      title: "Attention",42      scope: "serve",43      description: "Use the platform default unless another backend was measured end to end.",44      default: "platform",45      options: [{ id: "platform", label: "Platform default", recommended: true }],46    },47    {48      id: "precision",49      title: "Precision",50      scope: "serve",51      default: "native",52      options: [{ id: "native", label: "Native mixed precision", recommended: true }],53    },54    {55      id: "encoder",56      title: "Encoder",57      scope: "serve",58      default: "auto",59      options: [{ id: "auto", label: "Auto", flags: ["--encoder-parallel auto"], recommended: true }],60    },61    {62      id: "execution",63      title: "Execution",64      scope: "serve",65      default: "eager",66      options: [{ id: "eager", label: "Eager", recommended: true }],67    },68    {69      id: "quality",70      title: "Quality",71      scope: "request",72      default: "lossless",73      options: [{ id: "lossless", label: "Lossless", recommended: true }],74    },75    {76      id: "outputs",77      title: "Outputs",78      scope: "request",79      kind: "number",80      min: 1,81      max: 10,82      unit: "outputs per prompt",83      default: 1,84      options: [],85    },86  ],87 88  commandBuilder: {89    defaultSelection: {90      hw: "__DEFAULT_HW__",91      nodes: 1,92      gpus_per_node: __DEFAULT_GPUS__,93      topology_mode: "auto",94      tp_size: 1,95      ulysses_degree: __DEFAULT_GPUS__,96      ring_degree: 1,97    },98    resource: {99      limits: {100        nodes: { min: 1, max: __MAX_NODES__ },101        gpus_per_node: { min: 1, max: __MAX_GPUS_PER_NODE__ },102      },103      verifiedRecipes: [{104        id: "__DEFAULT_HW__-default",105        hw: "__DEFAULT_HW__",106        nodes: 1,107        gpus_per_node: __DEFAULT_GPUS__,108        placement: "resident",109        tp_size: 1,110        ulysses_degree: __DEFAULT_GPUS__,111        ring_degree: 1,112        encoder: "auto",113        default: true,114      }],115      autoTopology: (s) => ({116        tp_size: 1,117        ulysses_degree: Number(s.gpus_per_node),118        ring_degree: Number(s.nodes),119      }),120      validateTopology: (s, topology) => {121        const world = Number(s.nodes) * Number(s.gpus_per_node);122        const product = Number(topology.tp_size) * Number(topology.ulysses_degree) * Number(topology.ring_degree);123        return world === product ? [] : [`World size ${world} must equal TP × Ulysses × Ring (${product}).`];124      },125    },126    resolveDeployment: (s) => {127      const resource = config.commandBuilder.resource;128      const topology = s.topology_mode === "manual"129        ? { tp_size: Number(s.tp_size), ulysses_degree: Number(s.ulysses_degree), ring_degree: Number(s.ring_degree) }130        : resource.autoTopology(s);131      const errors = resource.validateTopology(s, topology);132      const recipe = resource.verifiedRecipes.find((entry) => entry.hw === s.hw133        && entry.nodes === Number(s.nodes)134        && entry.gpus_per_node === Number(s.gpus_per_node)135        && entry.placement === s.placement136        && entry.tp_size === topology.tp_size137        && entry.ulysses_degree === topology.ulysses_degree138        && entry.ring_degree === topology.ring_degree);139      const world = Number(s.nodes) * Number(s.gpus_per_node);140      const flags = ["--model-path {{MODEL_NAME}}", `--num-gpus ${world}`];141      if (topology.tp_size > 1) flags.push(`--tp-size ${topology.tp_size}`);142      flags.push(`--ulysses-degree ${topology.ulysses_degree}`);143      if (topology.ring_degree > 1) flags.push(`--ring-degree ${topology.ring_degree}`);144      flags.push("--host {{HOST_IP}}", "--port {{PORT}}");145      const verified = !!recipe && errors.length === 0;146      return {147        match: { hw: s.hw },148        nnodes: Number(s.nodes),149        verified,150        flags,151        builder: {152          topology,153          topologySummary: `TP ${topology.tp_size} · Ulysses ${topology.ulysses_degree} · Ring ${topology.ring_degree}`,154          errors,155          warnings: verified ? [] : ["Valid custom topology; exact end-to-end verification is pending."],156          verification: { serve: verified ? "verified" : "unverified", request: verified ? "verified" : "unverified" },157        },158      };159    },160  },161 162  modelNames: { default: "__MODEL_ID__" },163  placeholders: {164    HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" },165    PORT: { target: "command", label: "Bind port", default: "30010" },166    CURL_HOST: { target: "curl", label: "Server host", default: "localhost" },167    CURL_PORT: { target: "curl", label: "Server port", default: "30010" },168  },169  curl: (s) => `curl -sS -X POST http://{{CURL_HOST}}:{{CURL_PORT}}/__REQUEST_PATH__ \\170  -H 'Content-Type: application/json' \\171  -d '${JSON.stringify({ model: "{{MODEL_NAME}}", prompt: "__PROMPT__", quality: s.quality, num_outputs_per_prompt: Number(s.outputs) }, null, 2)}'`,172  cells: [],173};174