vllm.models.glm5next.common.model

_LOGIT_SCALE = 1.0 module-attribute

Output logit scale. A GLM-5.3-Flash trained value that neither the checkpoint nor Glm5NextTextConfig carries.

_MHC_POST_MULT_VALUE = 2.0 module-attribute

mHC post-multiplier. A GLM-5.3-Flash trained value that neither the checkpoint nor Glm5NextTextConfig carries.

_MHC_TAU = 0.05 module-attribute

mHC routing temperature. A GLM-5.3-Flash trained value that neither the checkpoint nor Glm5NextTextConfig carries.

_VISION_RMS_NORM_EPS = 1e-06 module-attribute

Vision tower RMSNorm epsilon.

GLM-5.3-Flash checkpoints ship vision_config.rms_norm_eps = 1e-5, but the vision tower was trained with 1e-6. Serving with 1e-5 drifts the RMSNorm and produces repetitive/degraded image descriptions, so force the trained value regardless of the checkpoint field.

_dequant_fp8_block(weight_fp8, scale_inv, block_size=128)

Dequantize a block-FP8 (e4m3) weight with per-block scale to BF16.

Unlike scaled_dequantize this tolerates a non-divisible (partial last block) shape by zero-padding to a multiple of block_size before the scale broadcast and trimming back afterwards (e.g. kv_a_proj_with_mqa is 576 rows = 4*128 + 64).

Source code in vllm/models/glm5next/common/model.py

def _dequant_fp8_block(
    weight_fp8: torch.Tensor,
    scale_inv: torch.Tensor,
    block_size: int = 128,
) -> torch.Tensor:
    """Dequantize a block-FP8 (e4m3) weight with per-block scale to BF16.

    Unlike ``scaled_dequantize`` this tolerates a non-divisible (partial last
    block) shape by zero-padding to a multiple of ``block_size`` before the
    scale broadcast and trimming back afterwards (e.g. kv_a_proj_with_mqa is
    576 rows = 4*128 + 64).
    """
    out_dim, in_dim = weight_fp8.shape
    pad_out = (-out_dim) % block_size
    pad_in = (-in_dim) % block_size
    w = weight_fp8
    if pad_out or pad_in:
        w = torch.nn.functional.pad(w, (0, pad_in, 0, pad_out))
    # scale_inv is (ceil(out/block), ceil(in/block)); broadcast to (out, in).
    s = scale_inv.to(torch.float32)
    s_full = s.repeat_interleave(block_size, dim=0).repeat_interleave(block_size, dim=1)
    out = (w.to(torch.float32) * s_full).to(torch.bfloat16)
    return out[:out_dim, :in_dim].contiguous()

_fused_shared_expert_name(name, n_routed_experts)

Point a checkpoint mlp.shared_experts.* tensor at the fused MoE's shared-expert slot, which follows the routed experts; other names are returned unchanged.

Source code in vllm/models/glm5next/common/model.py

def _fused_shared_expert_name(name: str, n_routed_experts: int) -> str:
    """Point a checkpoint ``mlp.shared_experts.*`` tensor at the fused MoE's
    shared-expert slot, which follows the routed experts; other names are
    returned unchanged."""
    return name.replace("mlp.shared_experts.", f"mlp.experts.{n_routed_experts}.", 1)

_fused_shared_experts_tuned(parallel_config)

AITER has fused-MoE configs tuned for the fused shared-expert shape (one more expert and one more top-k slot than the routed MoE) only on gfx950, with every expert on each rank and its weights split by TP4 or TP8. Data, prefill context and expert parallelism change that split, so any other GPU or parallel layout would run untuned fallback kernels.

Source code in vllm/models/glm5next/common/model.py

def _fused_shared_experts_tuned(parallel_config: ParallelConfig) -> bool:
    """AITER has fused-MoE configs tuned for the fused shared-expert shape
    (one more expert and one more top-k slot than the routed MoE) only on
    gfx950, with every expert on each rank and its weights split by TP4 or
    TP8. Data, prefill context and expert parallelism change that split, so
    any other GPU or parallel layout would run untuned fallback kernels."""
    from vllm.platforms.rocm import on_gfx950

    reasons: list[str] = []
    if not on_gfx950():
        reasons.append("the GPU is not gfx950")
    if parallel_config.tensor_parallel_size not in (4, 8):
        reasons.append(
            f"tensor_parallel_size is {parallel_config.tensor_parallel_size}"
        )
    if parallel_config.data_parallel_size != 1:
        reasons.append(f"data_parallel_size is {parallel_config.data_parallel_size}")
    if parallel_config.prefill_context_parallel_size != 1:
        reasons.append(
            "prefill_context_parallel_size is "
            f"{parallel_config.prefill_context_parallel_size}"
        )
    if parallel_config.enable_expert_parallel:
        reasons.append("expert parallelism is enabled")

    if not reasons:
        return True
    logger.warning_once(
        "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is ignored for GLM-5.3-Flash: "
        "%s. AITER has tuned configs for its fused shared-expert MoE only on "
        "gfx950 at TP4 and TP8, without data, prefill context or expert "
        "parallelism. Running the shared experts as a separate MLP.",
        "; ".join(reasons),
    )
    return False

_num_fused_shared_experts(n_shared_experts, enabled)

Expert slots the fused MoE appends for the shared expert; must match the num_fused_shared_experts that FusedMoE allocates.

Source code in vllm/models/glm5next/common/model.py

def _num_fused_shared_experts(n_shared_experts: int | None, enabled: bool) -> int:
    """Expert slots the fused MoE appends for the shared expert; must match the
    ``num_fused_shared_experts`` that ``FusedMoE`` allocates."""
    if not enabled or n_shared_experts is None:
        return 0
    if n_shared_experts > 1:
        raise NotImplementedError(
            "Fused shared-expert loading supports only 1 shared expert per "
            f"layer, but config.n_shared_experts is {n_shared_experts}. Set "
            "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 to run the shared "
            "experts as a separate MLP."
        )
    return n_shared_experts

_try_load_fp8_attn_proj(name, tensor, buf, params_dict, loaded_params, kv_a_pad_size)

Dequantize FP8 q_a_proj / kv_a_proj_with_mqa / o_proj to BF16 on load.

The FP8 checkpoint stores these as block-FP8 (weight + weight_scale_inv), but the model holds them in BF16 (fused_qkv_a_proj is always BF16 via DeepSeekV2FusedQkvAProjLinear; o_proj is excluded by modules_to_not_convert). When the model target is BF16 (no weight_scale_inv param) we dequantize; otherwise we return False so the normal stacked/direct path loads the FP8 tensor as-is.

Source code in vllm/models/glm5next/common/model.py

def _try_load_fp8_attn_proj(
    name,
    tensor,
    buf,
    params_dict,
    loaded_params,
    kv_a_pad_size: int,
) -> bool:
    """Dequantize FP8 q_a_proj / kv_a_proj_with_mqa / o_proj to BF16 on load.

    The FP8 checkpoint stores these as block-FP8 (weight + weight_scale_inv),
    but the model holds them in BF16 (``fused_qkv_a_proj`` is always BF16 via
    DeepSeekV2FusedQkvAProjLinear; ``o_proj`` is excluded by
    modules_to_not_convert). When the model target is BF16 (no
    ``weight_scale_inv`` param) we dequantize; otherwise we return False so the
    normal stacked/direct path loads the FP8 tensor as-is.
    """
    matched = None
    for suffix, info in _FP8_ATTN_PROJS.items():
        if suffix in name:
            matched = (suffix, info)
            break
    if matched is None:
        return False
    suffix, (key, target_base, shard_id, is_kva) = matched
    is_weight = name.endswith(".weight") and tensor.dtype == torch.float8_e4m3fn
    # Need to accept both the DeepSeek-native ``weight_scale_inv`` and the Quark
    # ``weight_scale`` names before feeding the shared block dequant below.
    is_scale = "weight_scale_inv" in name or name.endswith(".weight_scale")
    if not is_weight and not is_scale:
        return False

    layer_prefix = name.rsplit(suffix, 1)[0]
    target_w = f"{layer_prefix}.{target_base}.weight"
    target_s = f"{layer_prefix}.{target_base}.weight_scale_inv"
    # If the model actually kept this projection in FP8, let the normal path
    # handle it (it has a weight_scale_inv param).
    if target_s in params_dict:
        return False

    entry = buf.setdefault(layer_prefix, {}).setdefault(key, {})
    entry["weight" if is_weight else "scale"] = tensor
    if "weight" not in entry or "scale" not in entry:
        return True

    weight_fp8, scale_inv = entry["weight"], entry["scale"]
    buf[layer_prefix].pop(key, None)
    block_size = weight_fp8.shape[1] // scale_inv.shape[1]
    weight_bf16 = _dequant_fp8_block(weight_fp8, scale_inv, block_size)
    # NoPE: pad kv_a rope portion (kv_lora_rank -> kv_lora_rank + qk_rope_head_dim).
    if is_kva and kv_a_pad_size > 0:
        pad = torch.zeros(
            kv_a_pad_size,
            weight_bf16.shape[1],
            dtype=weight_bf16.dtype,
            device=weight_bf16.device,
        )
        weight_bf16 = torch.cat([weight_bf16, pad], dim=0)

    param = params_dict[target_w]
    if shard_id is None:
        param.weight_loader(param, weight_bf16)
    else:
        param.weight_loader(param, weight_bf16, shard_id)
    loaded_params.add(target_w)
    return True

_validate_supported_config(config)

Reject checkpoints using config options this implementation lacks.

The kpool indexer kernels always keep the incomplete trailing pool, so a checkpoint asking otherwise would be served silently wrong.

Source code in vllm/models/glm5next/common/model.py

def _validate_supported_config(config: Glm5NextTextConfig) -> None:
    """Reject checkpoints using config options this implementation lacks.

    The kpool indexer kernels always keep the incomplete trailing pool, so a
    checkpoint asking otherwise would be served silently wrong.
    """
    if config.index_topk is not None and not config.index_kpool_always_select_tail:
        raise NotImplementedError(
            "GLM-5.3 sparse indexer requires index_kpool_always_select_tail=True"
        )