sie_id: Qwen/Qwen3.6-27B hf_id: Qwen/Qwen3.6-27B hf_revision: 6a9e13bd6fc8f0983b9b99948120bc37f49c13e9 inputs: # Qwen3.6-27B is a unified vision-language model (Gated DeltaNet + Gated # Attention with an integrated vision encoder; same hybrid family as # Qwen3.5-4B but scaled to 64 layers / hidden_size=5120). The wire surface # accepts text+image via SIE-native generate (inline ``images``) and the # OpenAI chat-completions schema (``image_url`` content parts); video is # documented by Qwen but not yet wired through the SIE gateway. text: true image: true audio: false video: false tasks: generate: # Native context length is 262,144 tokens (YaRN extends to ~1M). # Keep the base model contract at the 8K launch window. The managed # tester rollout exposes the 32K surface through the # ``:rtx-pro-6000`` shape and its semantic ``:long-context`` alias below; # profile-level ``max_seq_length`` is promoted into each generated variant # config by the loader. context_length: 8192 max_output_tokens: 4096 # NEXTN cannot safely run constrained decoding. The gateway rewrites # grammar-constrained requests to this non-speculative sibling, which is # deployed on the same H100 machine class as the default lane. grammar_profile: no-spec capabilities: # EBNF remains unadvertised until it has model-specific runtime evidence. grammar: ["json_schema", "regex"] streaming: true tools: true # Validated on code + text-to-SQL — measured on H100 (bf16 no-spec, # greedy+min_tokens=10). BF16 benchmark: HumanEval 0.933, MBPP 0.81 # (beats the 4B code default 0.866/0.74), Spider exec-acc 0.693. # Informational (not request-gated); surfaces the high-end code/SQL option. code: true # CAVEAT: this flag is precision-agnostic, but FP8 weight quant regresses # SQL ~13pts (Spider 0.71 BF16 -> 0.58 FP8, same-subset control; see the # same-subset measurements). Code/tools/MC are FP8-safe. The RTX FP8 profile below is # for code/tools/general. Route SQL-critical traffic to the BF16 profile # or a BF16 bundle-qualified alias, e.g. # SIE_GATEWAY_MODEL_ALIASES={"sql":":/Qwen/Qwen3.6-27B"} # (gateway resolve_model_spec_with_aliases). sql: true # Qwen3.6 emits ``...`` reasoning by default. We # disable it for the OpenAI-compat path so visible output is the # answer only. Operators wanting CoT can flip this profile-side. chat_template_kwargs: enable_thinking: false prewarm_grammars: # Keep the prewarm pattern bare so it has the same portable semantics # across the worker preflight and SGLang's xgrammar runtime. - name: yes_no kind: regex value: "(yes|no)" - name: short_answer kind: json_schema value: type: object properties: answer: type: string required: [answer] max_sequence_length: 8192 # ── KV-cache math ── # # Qwen3.6-27B layer breakdown (per model card / config.json): # * 64 layers in a 16 × (3 × DeltaNet + 1 × GatedAttention) pattern # * 16 KV-bearing Gated-Attention layers (4 KV heads × head_dim=256) # * 48 recurrent Gated-DeltaNet layers — managed by SGLang's mamba # scheduler under ``--mamba-scheduler-strategy extra_buffer`` # # BF16 weights ≈ 27e9 × 2 B ≈ 54 GB before activations / KV. FP8 weight # quantization cuts the VRAM weight footprint to ~27 GB but still requires # enough host RAM to read the BF16 checkpoint before quantization. # * L4 (24 GB) → infeasible # * A100-40GB (40 GB) → infeasible BF16; FP8/TP2 only # * H100-80GB → BF16 speculative profiles, plus explicit non-spec # FP8, at the 8K launch window # * RTX PRO 6000 96GB → FP8 + NEXTN profile with a 32K variant surface profiles: # L4 / A100-40GB BF16 profiles intentionally omitted — Qwen3.6-27B's BF16 # weights (~54 GB) do not fit. default: max_batch_tokens: 16384 compute_precision: bfloat16 adapter_path: sie_server.adapters.sglang.generation:SGLangGenerationAdapter # H100-80GB launch shape at the 8K base-model context. NEXTN is # correctness-gated off on the pinned SGLang engine; retain the measured # load-time shape while the generic newer-engine lane is validated. The # managed cluster routes 32K 27B generation to ``rtx-pro-6000``. kv_budget_tokens: 8192 adapter_options: loadtime: mem_fraction_static: 0.95 served_model_name: Qwen/Qwen3.6-27B disable_cuda_graph: true attention_backend: triton grammar_backend: xgrammar reasoning_parser: qwen3 tool_call_parser: qwen3_coder # Preserve the smaller 2/2 H100 draft shape for a future validated # engine, but keep it disabled on the pinned SGLang release. speculative: enabled: false algorithm: nextn num_steps: 2 eagle_topk: 1 num_draft_tokens: 2 # On this conservative CUDA 12 profile, keep ``extra_buffer`` paired # with ``--disable-overlap-schedule``. The measured CUDA 13 EAGLE # native-window profiles below intentionally retain SPEC_V2 overlap # scheduling with the same mamba buffer strategy. extra_launch_args: - "--mm-process-config" - '{"image":{"min_pixels":65536,"max_pixels":1003520}}' - "--mamba-scheduler-strategy" - "extra_buffer" - "--disable-overlap-schedule" runtime: first_chunk_timeout_s: 90 inter_chunk_timeout_s: 15 overall_timeout_s: 600 default_sampling: temperature: 0.7 top_p: 0.8 top_k: 20 presence_penalty: 1.5 min_new_tokens: 10 stop_tokens: - "<|im_end|>" h100: max_batch_tokens: 32768 compute_precision: bfloat16 adapter_path: sie_server.adapters.sglang.generation:SGLangGenerationAdapter # Same H100-80GB target as ``default`` with the model-card 3/4 NEXTN draft # preserved from main. Keep the public H100 context at the 8K # base-model window until a separate 32K H100 smoke lands. # # Memory fit, measured on one H100 80GB with SGLang 0.5.10.post1: the # BF16 target weights take 51.05 GB of 77.98 GB. NEXTN reserves # intermediate mamba state for every running request and draft token # (about 0.14 GB per request per token), so SGLang's default of 48 # running requests needs about 27.5 GB and the pool cannot be sized at # any fraction. With 8 running requests at 0.93 the pool fits, but the # draft model then needs 5.53 GB (its embedding and head are loaded # before they are shared) with only 4.64 GB left. At 0.88 with 8 running # requests: mamba cache 42 slots (6.17 GB) plus 5.16 GB of speculative # state, a 113,971-token KV pool and the draft loaded with 2.54 GB to # spare. It served 8 concurrent requests, and a 7,027-token prompt # alongside 7 short ones. kv_budget_tokens: 32768 adapter_options: loadtime: mem_fraction_static: 0.88 served_model_name: Qwen/Qwen3.6-27B disable_cuda_graph: true attention_backend: triton grammar_backend: xgrammar reasoning_parser: qwen3 tool_call_parser: qwen3_coder speculative: enabled: true algorithm: nextn num_steps: 3 eagle_topk: 1 num_draft_tokens: 4 # ``--disable-overlap-schedule`` is the required pair-flag for # NEXTN + mamba-scheduler ``extra_buffer`` on the hybrid Gated- # DeltaNet architecture (same constraint as Qwen3.5-4B). extra_launch_args: - "--mm-process-config" - '{"image":{"min_pixels":65536,"max_pixels":1003520}}' - "--mamba-scheduler-strategy" - "extra_buffer" - "--disable-overlap-schedule" - "--max-running-requests" - "8" runtime: first_chunk_timeout_s: 90 inter_chunk_timeout_s: 15 overall_timeout_s: 600 default_sampling: temperature: 0.7 top_p: 0.8 top_k: 20 presence_penalty: 1.5 min_new_tokens: 10 stop_tokens: - "<|im_end|>" # H100-80GB FP8 profile for single-GPU EKS lanes. Keep the base model's # release-gated 8K context and disable NEXTN/CUDA graphs for a # correctness-first bootstrap. Online FP8 reduces the ~54 GB BF16 weights # to ~27 GB, leaving H100 headroom for activations and a conservative # two-admission KV budget. This is intentionally separate from ``h100`` # (BF16 + NEXTN) and ``rtx-pro-6000`` (FP8 + NEXTN + 32K). # Hardware placement remains separate: pair this model profile with the # cluster's H100-80GB machine profile. h100-fp8: # Inherit the grammar-safe, non-speculative profile so constrained # requests can keep this FP8 variant instead of falling back to BF16. extends: no-spec kv_budget_tokens: 16384 adapter_options: loadtime: mem_fraction_static: 0.90 served_model_name: Qwen/Qwen3.6-27B disable_cuda_graph: true attention_backend: triton grammar_backend: xgrammar reasoning_parser: qwen3 tool_call_parser: qwen3_coder speculative: enabled: false extra_launch_args: - "--mm-process-config" - '{"image":{"min_pixels":65536,"max_pixels":1003520}}' - "--quantization" - "fp8" # RTX PRO 6000 (96 GB GDDR7, Blackwell Server Edition, sm_120) profile. # FP8-first for max throughput: ``--quantization fp8`` (SGLang online # dynamic FP8 quant of the BF16 checkpoint) via the ``extra_launch_args`` # passthrough — ``compute_precision`` can only express the ``--dtype`` # axis (float16/bfloat16/float32), not the orthogonal ``--quantization`` # flag, so FP8 rides the same escape hatch already used for the mamba # scheduler. FP8 halves weight memory (~54 → ~27 GB), which frees room # for the *model-card* NEXTN 3/4 draft (num_steps=3 / num_draft_tokens=4) # that OOM'd ``init_memory_pool`` on H100-80GB even at mfs=0.97. The +16 GB # over H100 plus the FP8 weight saving is what makes 3/4 fit here. # # ACCURACY CONTRACT: FP8 is lossy. This profile is validated to within the # Wilson 95% CI of the *BF16* baseline on all four generation tasks. If FP8 # misses parity after bounded tuning, fall back to # BF16 + NEXTN 3/4 (drop the ``--quantization fp8`` pair below; the 96 GB # still fits the 3/4 draft in BF16). KV cache stays BF16 here — add # ``--kv-cache-dtype fp8_e4m3`` only if memory/throughput needs it AND # accuracy still holds (KV FP8 is usually the first thing to cost accuracy). # # Standalone block (no ``extends``): production ``resolve_profile`` does a # full-replace of ``loadtime`` for extending profiles, so a partial child # would drop inherited launch flags and desync the via-SIE path from the # bare-SGLang bench control. rtx-pro-6000: max_batch_tokens: 32768 compute_precision: bfloat16 adapter_path: sie_server.adapters.sglang.generation:SGLangGenerationAdapter # FP8 weights (~27 GB) + 96 GB total leaves KV/draft headroom for two # 32K-context admissions on the validated RTX-PRO-6000 serving path. kv_budget_tokens: 65536 adapter_options: loadtime: # Profile-scoped launch/context override. ``loadtime`` already # overrides adapter constructor kwargs; loader variant expansion # also promotes this into the generated ``:rtx-pro-6000`` model's # ``tasks.generate.context_length`` / ``max_sequence_length`` so # the 32K contract is attached to the RTX variant, not the base model. max_seq_length: 32768 # 0.90 is a conservative starting point — FP8 weights free enough # memory that the 3/4 NEXTN verification batch should fit with room # to spare. Iterate upward (smoke ``--mem-fraction-static``) once the # first boot confirms Blackwell sm_120 + FP8 GEMM kernels are present. mem_fraction_static: 0.90 served_model_name: Qwen/Qwen3.6-27B disable_cuda_graph: true # triton attention matches the rest of the Qwen3.x family. Blackwell # (sm_120) kernel coverage for triton + FP8 + NEXTN is the first thing # the boot smoke verifies; switch to flashinfer here if triton lacks # sm_120 coverage in the pinned SGLang build. attention_backend: triton grammar_backend: xgrammar reasoning_parser: qwen3 tool_call_parser: qwen3_coder # Model-card NEXTN 3/4 — restored here (vs ``default``'s conservative # 2/2) because FP8 + 96 GB fits the larger verification batch that # OOM'd on H100-80GB. speculative: enabled: true algorithm: nextn num_steps: 3 eagle_topk: 1 num_draft_tokens: 4 # ``--quantization fp8`` rides the passthrough (see header comment). # The mamba-scheduler + overlap pair-flags are the required NEXTN # companions on the hybrid Gated-DeltaNet architecture. List is the # FULL set (production full-replaces ``extra_launch_args``, not merge). extra_launch_args: - "--mm-process-config" - '{"image":{"min_pixels":65536,"max_pixels":1003520}}' - "--quantization" - "fp8" - "--mamba-scheduler-strategy" - "extra_buffer" - "--disable-overlap-schedule" runtime: first_chunk_timeout_s: 90 inter_chunk_timeout_s: 15 overall_timeout_s: 600 # Qwen3.6-27B empty-response fix baked in: under greedy/low-temp the # chat template emits EOS as the FIRST token on a large fraction of # prompts (n=50 6000 smoke: casehold 23/50, gpqa 29/50 came back # EMPTY). The floor ``min_new_tokens>=1`` fixes it — validated on the # 6000: min_tokens=10 → 0/50 empty on all four tasks, accuracy within # Wilson 95% CI of the BF16 baseline. ``min_new_tokens`` is the # SGLang-native key; the adapter merges this dict via ``setdefault``, # so a request-supplied ``min_tokens`` still wins. NOTE: this only # takes effect because ``runtime.default_sampling`` is now wired into # the adapter (core/loader.py); before that fix a key here was a # silent no-op and chat clients had to pass ``min_tokens`` themselves. default_sampling: temperature: 0.7 top_p: 0.8 top_k: 20 presence_penalty: 1.5 min_new_tokens: 10 stop_tokens: - "<|im_end|>" # Explicit native-window shape for one H100-80GB admission. Keep this # standalone (rather than extending ``h100-fp8``) so the matching # ``long-context-thinking`` child can inherit it without forbidden profile # chaining. The CUDA 13 engine's trained-in EAGLE draft, CUDA graphs, and # overlap schedule sustain the same batch-one decode rate at 32K, 128K, and # 256K. Online FP8 limits weight memory; KV remains BF16. long-context: # Qwen's official non-thinking serving recipe uses max_tokens=32768. # Keep the bare 8K profile at 4096 while exposing that decode ceiling only # on variants whose total context can accommodate it. max_output_tokens: 32768 max_batch_tokens: 262144 compute_precision: bfloat16 adapter_path: sie_server.adapters.sglang.cuda13:SGLangCuda13Adapter grammar_profile: long-context-no-spec kv_budget_tokens: 262144 adapter_options: loadtime: max_seq_length: 262144 mem_fraction_static: 0.90 served_model_name: Qwen/Qwen3.6-27B attention_backend: triton grammar_backend: xgrammar reasoning_parser: qwen3 tool_call_parser: qwen3_coder speculative: enabled: true algorithm: eagle num_steps: 3 eagle_topk: 1 num_draft_tokens: 4 extra_launch_args: - "--mm-process-config" - '{"image":{"min_pixels":65536,"max_pixels":1003520}}' - "--quantization" - "fp8" - "--mamba-radix-cache-strategy" - "extra_buffer" - "--page-size" - "64" runtime: first_chunk_timeout_s: 900 inter_chunk_timeout_s: 30 overall_timeout_s: 1800 default_sampling: temperature: 0.7 top_p: 0.8 top_k: 20 presence_penalty: 1.5 min_new_tokens: 10 stop_tokens: - "<|im_end|>" # Grammar-safe twin of the measured 256K H100 launch. It preserves every # load-time axis except EAGLE, which must be disabled for constrained decode. long-context-no-spec: extends: long-context adapter_options: loadtime: max_seq_length: 262144 mem_fraction_static: 0.90 served_model_name: Qwen/Qwen3.6-27B attention_backend: triton grammar_backend: xgrammar reasoning_parser: qwen3 tool_call_parser: qwen3_coder speculative: enabled: false extra_launch_args: - "--mm-process-config" - '{"image":{"min_pixels":65536,"max_pixels":1003520}}' - "--quantization" - "fp8" - "--mamba-radix-cache-strategy" - "extra_buffer" - "--page-size" - "64" # Keep thinking non-speculative for strict-reasoning correctness, but use the # separately measured CUDA-graph + overlap launch. Exact H100-80GB standalone # and gateway/NATS validation covered sustained churn, cache boundaries, # cancellation/recovery, and near-window prefill plus decode. This block is # standalone so its aliases inherit only the validated thinking shape. long-context-thinking: grammar_profile: long-context-thinking-no-spec # Official Qwen3.6 thinking examples use max_tokens=81920. max_output_tokens: 81920 max_batch_tokens: 262144 compute_precision: bfloat16 # SGLang 0.5.13's strict-thinking token filter prevents a premature EOS # from leaving the whole answer inside the private reasoning channel. adapter_path: sie_server.adapters.sglang.cuda13:SGLangStrictThinkingAdapter kv_budget_tokens: 262144 chat_template_kwargs: enable_thinking: true adapter_options: loadtime: max_seq_length: 262144 mem_fraction_static: 0.95 served_model_name: Qwen/Qwen3.6-27B attention_backend: triton grammar_backend: xgrammar reasoning_parser: qwen3 tool_call_parser: qwen3_coder speculative: enabled: false extra_launch_args: - "--mm-process-config" - '{"image":{"min_pixels":65536,"max_pixels":1003520}}' - "--quantization" - "fp8" - "--mamba-radix-cache-strategy" - "extra_buffer" - "--page-size" - "64" runtime: first_chunk_timeout_s: 900 inter_chunk_timeout_s: 30 overall_timeout_s: 1800 default_sampling: temperature: 1.0 top_p: 0.95 top_k: 20 presence_penalty: 0.0 min_new_tokens: 10 stop_tokens: - "<|im_end|>" # Dedicated constrained-decoding identity for the non-speculative thinking # launch. It is intentionally identical because the optimized parent also # remains non-speculative and grammar-safe. long-context-thinking-no-spec: extends: long-context-thinking # Hardware-explicit aliases for the exact H100-80GB native-window shape. # Machine placement remains a separate gateway decision; these names make # the validated context/GPU pairing discoverable to callers. h100-256k: extends: long-context grammar_profile: long-context-no-spec h100-256k-thinking: extends: long-context-thinking grammar_profile: long-context-thinking-no-spec # Explicit reasoning mode uses the validated native-window H100 lane. Qwen's # model card recommends at least 128K context for thinking; keeping this alias # on the 8K base lane would expose a mode that routinely truncates before its # private thought closes. The bare model remains non-thinking and answer-only. thinking: extends: long-context-thinking grammar_profile: long-context-thinking-no-spec # Throughput profile (H100-80GB) — the flagship's batch story. The # ``default`` NEXTN-spec shape is latency-optimized single-stream and # throughput-FLAT (~44 tok/s aggregate c=1→32, effective batch ≈1 — # MP.4 gen-smartest measurement): speculative verification + cuda-graphs # off + ``--disable-overlap-schedule`` serialize decode. This profile # mirrors what made Qwen3-4B-Instruct-2507 scale 36× (c=1→64) on its # spec-off default: speculative OFF, cuda graphs ON (no # ``--disable-cuda-graph``), overlap schedule ON (no # ``--disable-overlap-schedule``), so SGLang's continuous batcher can # actually run a deep decode batch. # # Launch-shape notes: # * No ``--mamba-scheduler-strategy extra_buffer``: that flag (and the # overlap disable) is the required NEXTN pair-flag set on the hybrid # Gated-DeltaNet arch; with spec off the adapter guard doesn't demand # it and the default mamba scheduler serves batch decode. # * ``mem_fraction_static: 0.90``: no NEXTN draft weights/KV to host, # but cuda-graph capture needs workspace outside the static pool — # 0.90 leaves headroom on top of the ~54 GB BF16 weights. # * ``--cuda-graph-max-bs 32``: pin graph coverage through the measured # c=32 operating point so deep decode batches stay on graphs. # * triton attention + xgrammar structured decoding. # # UX story: ``Qwen/Qwen3.6-27B`` (default, latency / single-stream) · # ``:batch`` (this profile, throughput) · ``:thinking`` (reasoning). # Standalone block (no ``extends``) — same rationale as ``rtx-pro-6000``. batch: max_batch_tokens: 32768 compute_precision: bfloat16 adapter_path: sie_server.adapters.sglang.generation:SGLangGenerationAdapter kv_budget_tokens: 32768 adapter_options: loadtime: mem_fraction_static: 0.90 served_model_name: Qwen/Qwen3.6-27B attention_backend: triton grammar_backend: xgrammar reasoning_parser: qwen3 tool_call_parser: qwen3_coder speculative: enabled: false extra_launch_args: - "--mm-process-config" - '{"image":{"min_pixels":65536,"max_pixels":1003520}}' - "--cuda-graph-max-bs" - "32" runtime: first_chunk_timeout_s: 90 inter_chunk_timeout_s: 15 overall_timeout_s: 600 default_sampling: temperature: 0.7 top_p: 0.8 top_k: 20 presence_penalty: 1.5 min_new_tokens: 10 stop_tokens: - "<|im_end|>" # No-speculative baseline — for SIE-vs-raw-SGLang ablation cells so # spec-decoding's contribution can be measured independently. Keeps # the same ``extra_launch_args`` as ``default`` / ``h100`` so a config # diff between them shows only the ``speculative`` block (the intent # of the ablation), matching Qwen3.5-4B's convention. no-spec: max_batch_tokens: 32768 compute_precision: bfloat16 adapter_path: sie_server.adapters.sglang.generation:SGLangGenerationAdapter kv_budget_tokens: 65536 adapter_options: loadtime: mem_fraction_static: 0.85 served_model_name: Qwen/Qwen3.6-27B disable_cuda_graph: true attention_backend: triton grammar_backend: xgrammar reasoning_parser: qwen3 tool_call_parser: qwen3_coder speculative: enabled: false extra_launch_args: - "--mm-process-config" - '{"image":{"min_pixels":65536,"max_pixels":1003520}}' - "--mamba-scheduler-strategy" - "extra_buffer" - "--disable-overlap-schedule" runtime: first_chunk_timeout_s: 90 inter_chunk_timeout_s: 15 overall_timeout_s: 600 # A strict grammar may complete valid JSON in fewer than ten tokens. # Do not force post-schema tokens on grammar-routed profiles. default_sampling: temperature: 0.7 top_p: 0.8 top_k: 20 presence_penalty: 1.5 stop_tokens: - "<|im_end|>"