|
| 1 | +# SPDX-License-Identifier: Apache-2.0 |
| 2 | +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | +import argparse |
| 4 | +import itertools |
| 5 | + |
| 6 | +import torch |
| 7 | + |
| 8 | +from vllm import _custom_ops as ops |
| 9 | +from vllm.model_executor.layers.fused_moe.moe_align_block_size import ( |
| 10 | + moe_align_block_size_triton, |
| 11 | +) |
| 12 | +from vllm.triton_utils import triton |
| 13 | + |
| 14 | + |
| 15 | +def get_topk_ids(num_tokens: int, num_experts: int, topk: int) -> torch.Tensor: |
| 16 | + return torch.stack( |
| 17 | + [ |
| 18 | + torch.randperm(num_experts, dtype=torch.int32, device="cuda")[:topk] |
| 19 | + for _ in range(num_tokens) |
| 20 | + ] |
| 21 | + ) |
| 22 | + |
| 23 | + |
| 24 | +def check_correctness(num_tokens, num_experts=256, block_size=256, topk=8): |
| 25 | + """ |
| 26 | + Verifies vllm vs. Triton |
| 27 | + """ |
| 28 | + topk_ids = get_topk_ids(num_tokens, num_experts, topk) |
| 29 | + |
| 30 | + # 1. malloc space for triton and vllm |
| 31 | + # malloc enough space (max_num_tokens_padded) for the sorted ids |
| 32 | + max_num_tokens_padded = topk_ids.numel() + num_experts * (block_size - 1) |
| 33 | + sorted_ids_triton = torch.empty( |
| 34 | + (max_num_tokens_padded,), dtype=torch.int32, device="cuda" |
| 35 | + ) |
| 36 | + sorted_ids_triton.fill_(topk_ids.numel()) # fill with sentinel value |
| 37 | + expert_ids_triton = torch.zeros( |
| 38 | + (max_num_tokens_padded // block_size,), dtype=torch.int32, device="cuda" |
| 39 | + ) |
| 40 | + num_tokens_post_pad_triton = torch.empty((1,), dtype=torch.int32, device="cuda") |
| 41 | + |
| 42 | + sorted_ids_vllm = torch.empty_like(sorted_ids_triton) |
| 43 | + sorted_ids_vllm.fill_(topk_ids.numel()) |
| 44 | + expert_ids_vllm = torch.zeros_like(expert_ids_triton) |
| 45 | + num_tokens_post_pad_vllm = torch.empty_like(num_tokens_post_pad_triton) |
| 46 | + |
| 47 | + # 2. run implementations |
| 48 | + moe_align_block_size_triton( |
| 49 | + topk_ids, |
| 50 | + num_experts, |
| 51 | + block_size, |
| 52 | + sorted_ids_triton, |
| 53 | + expert_ids_triton, |
| 54 | + num_tokens_post_pad_triton, |
| 55 | + ) |
| 56 | + |
| 57 | + ops.moe_align_block_size( |
| 58 | + topk_ids, |
| 59 | + num_experts, |
| 60 | + block_size, |
| 61 | + sorted_ids_vllm, |
| 62 | + expert_ids_vllm, |
| 63 | + num_tokens_post_pad_vllm, |
| 64 | + ) |
| 65 | + print(f"✅ VLLM implementation works with {num_experts} experts!") |
| 66 | + |
| 67 | + # 3. compare results |
| 68 | + if torch.allclose(expert_ids_triton, expert_ids_vllm) and torch.allclose( |
| 69 | + num_tokens_post_pad_triton, num_tokens_post_pad_vllm |
| 70 | + ): |
| 71 | + print("✅ Triton and VLLM implementations match.") |
| 72 | + else: |
| 73 | + print("❌ Triton and VLLM implementations DO NOT match.") |
| 74 | + print("Triton expert_ids:", expert_ids_triton) |
| 75 | + print("VLLM expert_ids:", expert_ids_vllm) |
| 76 | + print("Triton num_tokens_post_pad:", num_tokens_post_pad_triton) |
| 77 | + print("VLLM num_tokens_post_pad:", num_tokens_post_pad_vllm) |
| 78 | + |
| 79 | + |
| 80 | +# test configurations |
| 81 | +num_tokens_range = [1, 16, 256, 4096] |
| 82 | +num_experts_range = [16, 64, 224, 256, 280, 512] |
| 83 | +topk_range = [1, 2, 8] |
| 84 | +configs = list(itertools.product(num_tokens_range, num_experts_range, topk_range)) |
| 85 | + |
| 86 | + |
| 87 | +@triton.testing.perf_report( |
| 88 | + triton.testing.Benchmark( |
| 89 | + x_names=["num_tokens", "num_experts", "topk"], |
| 90 | + x_vals=configs, |
| 91 | + line_arg="provider", |
| 92 | + line_vals=["vllm", "triton"], # "triton" |
| 93 | + line_names=["VLLM", "Triton"], # "Triton" |
| 94 | + plot_name="moe-align-block-size-performance", |
| 95 | + args={}, |
| 96 | + ) |
| 97 | +) |
| 98 | +def benchmark(num_tokens, num_experts, topk, provider): |
| 99 | + """Benchmark function for Triton.""" |
| 100 | + block_size = 256 |
| 101 | + topk_ids = get_topk_ids(num_tokens, num_experts, topk) |
| 102 | + |
| 103 | + max_num_tokens_padded = topk_ids.numel() + num_experts * (block_size - 1) |
| 104 | + sorted_ids = torch.empty((max_num_tokens_padded,), dtype=torch.int32, device="cuda") |
| 105 | + sorted_ids.fill_(topk_ids.numel()) |
| 106 | + max_num_m_blocks = max_num_tokens_padded // block_size |
| 107 | + expert_ids = torch.empty((max_num_m_blocks,), dtype=torch.int32, device="cuda") |
| 108 | + num_tokens_post_pad = torch.empty((1,), dtype=torch.int32, device="cuda") |
| 109 | + |
| 110 | + quantiles = [0.5, 0.2, 0.8] |
| 111 | + |
| 112 | + if provider == "vllm": |
| 113 | + ms, min_ms, max_ms = triton.testing.do_bench( |
| 114 | + lambda: ops.moe_align_block_size( |
| 115 | + topk_ids, |
| 116 | + num_experts, |
| 117 | + block_size, |
| 118 | + sorted_ids.clone(), |
| 119 | + expert_ids.clone(), |
| 120 | + num_tokens_post_pad.clone(), |
| 121 | + ), |
| 122 | + quantiles=quantiles, |
| 123 | + ) |
| 124 | + elif provider == "triton": |
| 125 | + ms, min_ms, max_ms = triton.testing.do_bench( |
| 126 | + lambda: moe_align_block_size_triton( |
| 127 | + topk_ids, |
| 128 | + num_experts, |
| 129 | + block_size, |
| 130 | + sorted_ids.clone(), |
| 131 | + expert_ids.clone(), |
| 132 | + num_tokens_post_pad.clone(), |
| 133 | + ), |
| 134 | + quantiles=quantiles, |
| 135 | + ) |
| 136 | + |
| 137 | + return 1000 * ms, 1000 * max_ms, 1000 * min_ms |
| 138 | + |
| 139 | + |
| 140 | +if __name__ == "__main__": |
| 141 | + parser = argparse.ArgumentParser() |
| 142 | + parser.add_argument( |
| 143 | + "--num_experts", |
| 144 | + type=int, |
| 145 | + default=64, |
| 146 | + choices=[8, 16, 32, 64, 128, 256], |
| 147 | + ) |
| 148 | + parser.add_argument( |
| 149 | + "--topk", |
| 150 | + type=int, |
| 151 | + default=8, |
| 152 | + choices=[2, 4, 8], |
| 153 | + help="Top-k value for correctness check.", |
| 154 | + ) |
| 155 | + args = parser.parse_args() |
| 156 | + |
| 157 | + print("Running correctness check...") |
| 158 | + check_correctness(num_tokens=1024, num_experts=args.num_experts, topk=args.topk) |
| 159 | + benchmark.run(print_data=True, show_plots=True) |
0 commit comments