From 4ad2759fe3bddc2b15e462caea4ab9de9fa1e05a Mon Sep 17 00:00:00 2001 From: xuqianmamba <60861262+xuqianmamba@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:10:36 +0800 Subject: [PATCH] Add DeepSeek-V4 H20 simulation data --- README.md | 3 +- bench_data/dsv4/README.md | 32 +++ bench_data/dsv4/decode/h20/attention.csv | 3 + bench_data/dsv4/decode/h20/mhc.csv | 2 + bench_data/dsv4/decode/h20/moe.csv | 2 + bench_data/dsv4/prefill/h20/attention.csv | 3 + bench_data/dsv4/prefill/h20/mhc.csv | 2 + bench_data/dsv4/prefill/h20/moe.csv | 2 + config/model_config.py | 138 ++++++++- kernel_sim/dsv4.py | 263 ++++++++++++++++++ main.py | 101 ++++++- models/deepseek_v4_model.py | 236 ++++++++++++++++ .../bench_data/dsv4/decode/h20/attention.csv | 3 + .../bench_data/dsv4/decode/h20/mhc.csv | 2 + .../bench_data/dsv4/decode/h20/moe.csv | 2 + .../bench_data/dsv4/prefill/h20/attention.csv | 3 + .../bench_data/dsv4/prefill/h20/mhc.csv | 2 + .../bench_data/dsv4/prefill/h20/moe.csv | 2 + tests/fixtures/deepseek_v4/config.json | 23 ++ tests/test_deepseek_v4_simulator.py | 111 ++++++++ 20 files changed, 923 insertions(+), 12 deletions(-) create mode 100644 bench_data/dsv4/README.md create mode 100644 bench_data/dsv4/decode/h20/attention.csv create mode 100644 bench_data/dsv4/decode/h20/mhc.csv create mode 100644 bench_data/dsv4/decode/h20/moe.csv create mode 100644 bench_data/dsv4/prefill/h20/attention.csv create mode 100644 bench_data/dsv4/prefill/h20/mhc.csv create mode 100644 bench_data/dsv4/prefill/h20/moe.csv create mode 100644 kernel_sim/dsv4.py create mode 100644 models/deepseek_v4_model.py create mode 100644 tests/fixtures/deepseek_v4/bench_data/dsv4/decode/h20/attention.csv create mode 100644 tests/fixtures/deepseek_v4/bench_data/dsv4/decode/h20/mhc.csv create mode 100644 tests/fixtures/deepseek_v4/bench_data/dsv4/decode/h20/moe.csv create mode 100644 tests/fixtures/deepseek_v4/bench_data/dsv4/prefill/h20/attention.csv create mode 100644 tests/fixtures/deepseek_v4/bench_data/dsv4/prefill/h20/mhc.csv create mode 100644 tests/fixtures/deepseek_v4/bench_data/dsv4/prefill/h20/moe.csv create mode 100644 tests/fixtures/deepseek_v4/config.json create mode 100644 tests/test_deepseek_v4_simulator.py diff --git a/README.md b/README.md index 0166e4f..b4e88d9 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ The accuracy of simulation results relies heavily on the kernel benchmark result ## Supported Features -- **Attention**: MHA/GQA, MLA. Benchmarked on FlashInfer, FlashAttention-3, FlashMLA. +- **Attention**: MHA/GQA, MLA, and table-driven DeepSeek-V4 C4/C128 + compressed MQA. Benchmarked on FlashInfer, FlashAttention-3, FlashMLA. - **MoE**: GroupedGEMM. Benchmarked on DeepGEMM. - **Linear**: GEMM. Benchmarked on DeepGEMM. - **Parallelization**: DP Attn, EP MoE. diff --git a/bench_data/dsv4/README.md b/bench_data/dsv4/README.md new file mode 100644 index 0000000..34173d9 --- /dev/null +++ b/bench_data/dsv4/README.md @@ -0,0 +1,32 @@ +# DeepSeek-V4-Pro H20 TP8 measurements + +These rows were collected for architecture fingerprint +`dc8a15770f7f8647` on one node with eight NVIDIA H20 GPUs and SGLang +0.5.15. + +Each value is a per-base-decoder-layer CUDA-event median. For each timed +forward, latency is first reduced to the maximum across TP ranks, then the +median is taken across timed forwards. Communication is excluded and modeled +separately by InferSim. + +The serving keys are: + +- attention: `dsv4`, FP8 weights, `fp8_e4m3` KV cache and FP32 compression + state; +- mHC: `fused_post_pre`; +- MoE: native packed-FP4 experts with `flashinfer_mxfp4`. + +The exact measured shapes are prefill `B=1, Q=4096` and decode +`B=128, Q=1, past=5120`. InferSim intentionally rejects other fingerprints, +parallel configurations, precisions and shapes instead of silently +extrapolating these values. + +Run the matching end-to-end simulation with: + +```bash +python3 main.py \ + --config-path /path/to/DeepSeek-V4-Pro/config.json \ + --device-type H20 --world-size 8 --tp-size 8 --num-nodes 1 \ + --max-prefill-tokens 4096 --target-isl 4096 --target-osl 2048 \ + --decode-bs 128 --dsv4-decode-past-len 5120 +``` diff --git a/bench_data/dsv4/decode/h20/attention.csv b/bench_data/dsv4/decode/h20/attention.csv new file mode 100644 index 0000000..ae8ad9b --- /dev/null +++ b/bench_data/dsv4/decode/h20/attention.csv @@ -0,0 +1,3 @@ +architecture_fingerprint,tp_size,backend,attention_weight_dtype,kv_dtype,compress_state_dtype,ratio,batch_size,q_len,past_len,latency_us +dc8a15770f7f8647,8,dsv4,fp8,fp8_e4m3,fp32,128,128,1,5120,895.235094 +dc8a15770f7f8647,8,dsv4,fp8,fp8_e4m3,fp32,4,128,1,5120,1237.978661 diff --git a/bench_data/dsv4/decode/h20/mhc.csv b/bench_data/dsv4/decode/h20/mhc.csv new file mode 100644 index 0000000..d7a71bd --- /dev/null +++ b/bench_data/dsv4/decode/h20/mhc.csv @@ -0,0 +1,2 @@ +architecture_fingerprint,tp_size,backend,attention_weight_dtype,batch_size,q_len,mode,latency_us +dc8a15770f7f8647,8,dsv4,fp8,128,1,fused_post_pre,249.299934 diff --git a/bench_data/dsv4/decode/h20/moe.csv b/bench_data/dsv4/decode/h20/moe.csv new file mode 100644 index 0000000..0ee4546 --- /dev/null +++ b/bench_data/dsv4/decode/h20/moe.csv @@ -0,0 +1,2 @@ +architecture_fingerprint,tp_size,world_size,num_nodes,backend,expert_weight_dtype,batch_size,q_len,latency_us +dc8a15770f7f8647,8,8,1,flashinfer_mxfp4,fp4,128,1,1372.636325 diff --git a/bench_data/dsv4/prefill/h20/attention.csv b/bench_data/dsv4/prefill/h20/attention.csv new file mode 100644 index 0000000..1ddb1e6 --- /dev/null +++ b/bench_data/dsv4/prefill/h20/attention.csv @@ -0,0 +1,3 @@ +architecture_fingerprint,tp_size,backend,attention_weight_dtype,kv_dtype,compress_state_dtype,ratio,batch_size,q_len,past_len,latency_us +dc8a15770f7f8647,8,dsv4,fp8,fp8_e4m3,fp32,128,1,4096,0,3988.013406 +dc8a15770f7f8647,8,dsv4,fp8,fp8_e4m3,fp32,4,1,4096,0,7267.319441 diff --git a/bench_data/dsv4/prefill/h20/mhc.csv b/bench_data/dsv4/prefill/h20/mhc.csv new file mode 100644 index 0000000..a2e94a6 --- /dev/null +++ b/bench_data/dsv4/prefill/h20/mhc.csv @@ -0,0 +1,2 @@ +architecture_fingerprint,tp_size,backend,attention_weight_dtype,batch_size,q_len,mode,latency_us +dc8a15770f7f8647,8,dsv4,fp8,1,4096,fused_post_pre,778.451410 diff --git a/bench_data/dsv4/prefill/h20/moe.csv b/bench_data/dsv4/prefill/h20/moe.csv new file mode 100644 index 0000000..c680212 --- /dev/null +++ b/bench_data/dsv4/prefill/h20/moe.csv @@ -0,0 +1,2 @@ +architecture_fingerprint,tp_size,world_size,num_nodes,backend,expert_weight_dtype,batch_size,q_len,latency_us +dc8a15770f7f8647,8,8,1,flashinfer_mxfp4,fp4,1,4096,8966.938213 diff --git a/config/model_config.py b/config/model_config.py index 172de54..df5ce0c 100644 --- a/config/model_config.py +++ b/config/model_config.py @@ -1,4 +1,60 @@ +import hashlib import json +from dataclasses import dataclass + + +@dataclass(frozen=True) +class AttentionLayerGroup: + """A consecutive run of DeepSeek-V4 attention layers.""" + + compress_ratio: int + num_layers: int + + +def _normalize_dtype(value): + value = str(value).lower() + if value in {"fp8", "e4m3", "e5m2", "float8_e4m3fn", "float8_e5m2"}: + return "fp8" + if value in {"bf16", "bfloat16"}: + return "bf16" + if value in {"fp4", "nvfp4", "mxfp4"}: + return "fp4" + return value + + +def get_deepseek_v4_architecture_fingerprint(config): + """Return a stable key for benchmark rows belonging to one V4 variant.""" + if not config.is_deepseek_v4: + raise ValueError( + "DeepSeek-V4 architecture fingerprint requested for another model" + ) + architecture = { + "model_type": config.model_type, + "hidden_size": config.hidden_size, + "num_hidden_layers": config.num_hidden_layers, + "compress_ratios": config.compress_ratios, + "num_attention_heads": config.num_attention_heads, + "num_key_value_heads": config.num_key_value_heads, + "head_dim": config.head_dim, + "q_lora_rank": config.q_lora_rank, + "o_lora_rank": config.o_lora_rank, + "o_groups": config.num_output_groups, + "qk_rope_head_dim": config.qk_rope_head_dim, + "index_n_heads": config.index_n_heads, + "index_head_dim": config.index_head_dim, + "index_topk": config.index_topk, + "sliding_window": config.sliding_window, + "hc_mult": config.hc_mult, + "hc_sinkhorn_iters": config.hc_sinkhorn_iters, + "routed_experts": config.num_routed_experts, + "shared_experts": config.num_shared_experts, + "experts_per_tok": config.num_experts_per_tok, + "moe_intermediate_size": config.intermediate_size, + "attention_weight_dtype": config.attention_weight_dtype, + "expert_dtype": config.expert_dtype, + } + serialized = json.dumps(architecture, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(serialized.encode("utf-8")).hexdigest()[:16] class ModelConfig: @@ -15,9 +71,9 @@ def __init__( d = d["text_config"] self.model_type = d["model_type"] + self.is_deepseek_v4 = self.model_type == "deepseek_v4" self.is_qwen3_5_moe = ( - root_model_type == "qwen3_5_moe" - and self.model_type == "qwen3_5_moe_text" + root_model_type == "qwen3_5_moe" and self.model_type == "qwen3_5_moe_text" ) self.hidden_size = d["hidden_size"] @@ -41,7 +97,9 @@ def __init__( self.linear_num_value_heads = d["linear_num_value_heads"] self.attn_type = "MHA/GQA" - if "kv_lora_rank" in d: + if self.is_deepseek_v4: + self.attn_type = "DSV4" + elif "kv_lora_rank" in d: self.attn_type = "MLA" # attn @@ -62,11 +120,60 @@ def __init__( self.v_head_dim = d["v_head_dim"] self.index_topk = d.get("index_topk") self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim + elif self.attn_type == "DSV4": + self.num_attention_heads = d["num_attention_heads"] + self.num_key_value_heads = d["num_key_value_heads"] + self.head_dim = d["head_dim"] + self.q_lora_rank = d["q_lora_rank"] + self.o_lora_rank = d["o_lora_rank"] + self.num_output_groups = d["o_groups"] + self.qk_rope_head_dim = d["qk_rope_head_dim"] + self.qk_nope_head_dim = self.head_dim - self.qk_rope_head_dim + self.index_topk = d["index_topk"] + self.index_n_heads = d.get("index_n_heads", 64) + self.index_head_dim = d.get("index_head_dim", 128) + self.sliding_window = d.get("sliding_window", 128) + self.hc_mult = d.get("hc_mult", 4) + self.hc_sinkhorn_iters = d.get("hc_sinkhorn_iters", 20) + quantization_config = d.get("quantization_config", {}) + self.attention_weight_dtype = _normalize_dtype( + d.get( + "attention_weight_dtype", + quantization_config.get("fmt", d.get("torch_dtype", "bf16")), + ) + ) + self.expert_dtype = _normalize_dtype(d.get("expert_dtype", "bf16")) + + all_compress_ratios = list(d["compress_ratios"]) + if len(all_compress_ratios) < self.num_hidden_layers: + raise ValueError( + "DeepSeek-V4 compress_ratios must contain one entry for " + "each transformer layer" + ) + self.compress_ratios = all_compress_ratios[: self.num_hidden_layers] + self.nextn_compress_ratios = all_compress_ratios[self.num_hidden_layers :] + invalid_ratios = set(self.compress_ratios).difference({0, 4, 128}) + if invalid_ratios: + raise ValueError( + "Unsupported DeepSeek-V4 compression ratio(s): " + + ", ".join(str(ratio) for ratio in sorted(invalid_ratios)) + ) + self.attention_layer_groups = tuple( + self._group_compress_ratios(self.compress_ratios) + ) + self.attention_ratio_counts = {} + for group in self.attention_layer_groups: + self.attention_ratio_counts[group.compress_ratio] = ( + self.attention_ratio_counts.get(group.compress_ratio, 0) + + group.num_layers + ) # FFN/MoE self.is_moe = True if "num_routed_experts" in d: self.num_routed_experts = d["num_routed_experts"] + elif "n_routed_experts" in d: + self.num_routed_experts = d["n_routed_experts"] elif "num_experts" in d: self.num_routed_experts = d["num_experts"] else: @@ -89,7 +196,9 @@ def __init__( self.shared_expert_intermediate_size // self.intermediate_size ) else: - self.num_shared_experts = d.get("num_shared_experts", 0) + self.num_shared_experts = d.get( + "num_shared_experts", d.get("n_shared_experts", 0) + ) self.shared_expert_intermediate_size = ( self.num_shared_experts * self.intermediate_size ) @@ -98,3 +207,24 @@ def __init__( self.intermediate_size = d["intermediate_size"] self.num_shared_experts = 0 self.shared_expert_intermediate_size = 0 + + if self.is_deepseek_v4: + self.dsv4_architecture_fingerprint = ( + get_deepseek_v4_architecture_fingerprint(self) + ) + + @staticmethod + def _group_compress_ratios(compress_ratios): + if not compress_ratios: + return [] + groups = [] + current_ratio = compress_ratios[0] + count = 0 + for ratio in compress_ratios: + if ratio != current_ratio: + groups.append(AttentionLayerGroup(current_ratio, count)) + current_ratio = ratio + count = 0 + count += 1 + groups.append(AttentionLayerGroup(current_ratio, count)) + return groups diff --git a/kernel_sim/dsv4.py b/kernel_sim/dsv4.py new file mode 100644 index 0000000..c3e96eb --- /dev/null +++ b/kernel_sim/dsv4.py @@ -0,0 +1,263 @@ +"""Exact measured-kernel lookup for DeepSeek-V4.""" + +from __future__ import annotations + +import csv +from pathlib import Path + + +class MissingDSV4BenchmarkData(RuntimeError): + """Raised when a V4 simulation has no exact measured row.""" + + +def _normalize(value): + return str(value).strip().lower() + + +def canonical_dsv4_attention_backend(value): + value = _normalize(value) + return { + "compressed": "dsv4", + "sglang_hopper": "dsv4", + }.get(value, value) + + +def canonical_dsv4_kv_dtype(value): + value = _normalize(value) + aliases = { + "fp8": "fp8_e4m3", + "e4m3": "fp8_e4m3", + "fp8_e4m3": "fp8_e4m3", + "fp8_e4m3fn": "fp8_e4m3", + "float8_e4m3fn": "fp8_e4m3", + "bf16": "bfloat16", + "bfloat16": "bfloat16", + } + try: + return aliases[value] + except KeyError as error: + raise ValueError( + "DeepSeek-V4 KV dtype must be fp8_e4m3 or bfloat16; " f"got {value!r}" + ) from error + + +def canonical_dsv4_moe_key(backend, expert_weight_dtype): + backend = _normalize(backend) + backend = { + "marlin_w4a16": "marlin", + "w4a16_marlin": "marlin", + }.get(backend, backend) + expert_weight_dtype = _normalize(expert_weight_dtype) + if backend == "auto": + raise ValueError( + "DeepSeek-V4 benchmark rows require the resolved MoE backend, " "not 'auto'" + ) + if backend == "flashinfer_mxfp4" and expert_weight_dtype != "fp4": + raise ValueError("flashinfer_mxfp4 benchmark rows require fp4 expert weights") + return backend, expert_weight_dtype + + +class DSV4BenchmarkData: + """Read exact V4 layer-component latencies from ``bench_data/dsv4``.""" + + _REQUIRED_COLUMNS = { + "attention.csv": { + "architecture_fingerprint", + "tp_size", + "backend", + "attention_weight_dtype", + "kv_dtype", + "compress_state_dtype", + "ratio", + "batch_size", + "q_len", + "past_len", + "latency_us", + }, + "mhc.csv": { + "architecture_fingerprint", + "tp_size", + "backend", + "attention_weight_dtype", + "batch_size", + "q_len", + "mode", + "latency_us", + }, + "moe.csv": { + "architecture_fingerprint", + "tp_size", + "world_size", + "num_nodes", + "backend", + "expert_weight_dtype", + "batch_size", + "q_len", + "latency_us", + }, + } + + def __init__(self, root=None, architecture_fingerprint=None): + if root is None: + root = Path(__file__).resolve().parents[1] / "bench_data" / "dsv4" + self.root = Path(root) + self.architecture_fingerprint = architecture_fingerprint + self._cache = {} + + def _path(self, phase, device_type, file_name): + return self.root / phase / device_type.lower() / file_name + + @staticmethod + def _canonicalize(file_name, values): + values = dict(values) + if file_name in {"attention.csv", "mhc.csv"}: + values["backend"] = canonical_dsv4_attention_backend(values["backend"]) + if file_name == "attention.csv": + values["kv_dtype"] = canonical_dsv4_kv_dtype(values["kv_dtype"]) + if file_name == "moe.csv": + ( + values["backend"], + values["expert_weight_dtype"], + ) = canonical_dsv4_moe_key(values["backend"], values["expert_weight_dtype"]) + return values + + def _read_rows(self, phase, device_type, file_name): + path = self._path(phase, device_type, file_name) + if path in self._cache: + return self._cache[path] + if not path.exists(): + raise MissingDSV4BenchmarkData(f"Missing benchmark file: {path}") + + with path.open(newline="", encoding="utf-8") as csv_file: + reader = csv.DictReader(csv_file) + fields = set(reader.fieldnames or []) + missing = self._REQUIRED_COLUMNS[file_name] - fields + if missing: + raise MissingDSV4BenchmarkData( + f"{path} is missing columns: {', '.join(sorted(missing))}" + ) + rows = [ + self._canonicalize(file_name, row) + for row in reader + if any(row.values()) + ] + self._cache[path] = rows + return rows + + def _exact_latency(self, phase, device_type, file_name, filters): + path = self._path(phase, device_type, file_name) + filters = self._canonicalize(file_name, filters) + if self.architecture_fingerprint is not None: + filters["architecture_fingerprint"] = self.architecture_fingerprint + + matches = [ + row + for row in self._read_rows(phase, device_type, file_name) + if all( + str(row.get(key, "")) == str(value) for key, value in filters.items() + ) + ] + description = ", ".join(f"{key}={value}" for key, value in filters.items()) + if len(matches) != 1: + raise MissingDSV4BenchmarkData( + f"Expected one exact row in {path} for {description}; " + f"found {len(matches)}. DeepSeek-V4 does not use a " + "nearest-shape or analytical fallback." + ) + try: + latency_us = float(matches[0]["latency_us"]) + except (TypeError, ValueError) as error: + raise MissingDSV4BenchmarkData( + f"Invalid latency_us in {path}: {matches[0].get('latency_us')!r}" + ) from error + if latency_us <= 0: + raise MissingDSV4BenchmarkData( + f"latency_us in {path} must be positive, got {latency_us}" + ) + return latency_us + + def attention_latency_us( + self, + *, + phase, + device_type, + tp_size, + backend, + attention_weight_dtype, + kv_dtype, + compress_state_dtype, + ratio, + batch_size, + q_len, + past_len, + ): + return self._exact_latency( + phase, + device_type, + "attention.csv", + { + "tp_size": tp_size, + "backend": backend, + "attention_weight_dtype": attention_weight_dtype, + "kv_dtype": kv_dtype, + "compress_state_dtype": compress_state_dtype, + "ratio": ratio, + "batch_size": batch_size, + "q_len": q_len, + "past_len": past_len, + }, + ) + + def mhc_latency_us( + self, + *, + phase, + device_type, + tp_size, + backend, + attention_weight_dtype, + batch_size, + q_len, + mode, + ): + return self._exact_latency( + phase, + device_type, + "mhc.csv", + { + "tp_size": tp_size, + "backend": backend, + "attention_weight_dtype": attention_weight_dtype, + "batch_size": batch_size, + "q_len": q_len, + "mode": mode, + }, + ) + + def moe_latency_us( + self, + *, + phase, + device_type, + tp_size, + world_size, + num_nodes, + backend, + expert_weight_dtype, + batch_size, + q_len, + ): + return self._exact_latency( + phase, + device_type, + "moe.csv", + { + "tp_size": tp_size, + "world_size": world_size, + "num_nodes": num_nodes, + "backend": backend, + "expert_weight_dtype": expert_weight_dtype, + "batch_size": batch_size, + "q_len": q_len, + }, + ) diff --git a/main.py b/main.py index e71c9bd..c1cf3b2 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,8 @@ import argparse from config.model_config import ModelConfig +from kernel_sim.dsv4 import canonical_dsv4_kv_dtype +from models.deepseek_v4_model import DeepSeekV4Model from models.hybrid_model import HybridModel from models.model import Model @@ -15,16 +17,16 @@ def mfu_value(value): def non_negative_float(value): value = float(value) if not 0 <= value < float("inf"): - raise argparse.ArgumentTypeError("value must be finite and greater than or equal to 0") + raise argparse.ArgumentTypeError( + "value must be finite and greater than or equal to 0" + ) return value def main(args): config = ModelConfig(args.config_path) - enable_shared_expert_overlap = getattr( - args, "enable_shared_expert_overlap", False - ) + enable_shared_expert_overlap = getattr(args, "enable_shared_expert_overlap", False) if enable_shared_expert_overlap and not ( config.is_qwen3_5_moe and config.is_hybrid_linear ): @@ -38,8 +40,27 @@ def main(args): print("{:<40} {:<10}".format("World size:", args.world_size)) print("{:<40} {:<10}".format("TP size:", args.tp_size)) print("{:<40} {:<10}".format("Attn type:", config.attn_type)) - print("{:<40} {:<10}".format("Use FP8 GEMM:", args.use_fp8_gemm)) - print("{:<40} {:<10}".format("Use FP8 KV:", args.use_fp8_kv)) + if config.is_deepseek_v4: + print( + "{:<40} {:<10}".format( + "V4 attention weight dtype:", + args.dsv4_attention_weight_dtype or config.attention_weight_dtype, + ) + ) + print( + "{:<40} {:<10}".format( + "V4 expert weight dtype:", + args.dsv4_expert_weight_dtype or config.expert_dtype, + ) + ) + print( + "{:<40} {:<10}".format( + "V4 KV dtype:", canonical_dsv4_kv_dtype(args.dsv4_kv_dtype) + ) + ) + else: + print("{:<40} {:<10}".format("Use FP8 GEMM:", args.use_fp8_gemm)) + print("{:<40} {:<10}".format("Use FP8 KV:", args.use_fp8_kv)) if config.is_qwen3_5_moe and config.is_hybrid_linear: print( "{:<40} {:<10}".format( @@ -47,7 +68,10 @@ def main(args): ) ) - if config.is_hybrid_linear: + if config.is_deepseek_v4: + model = DeepSeekV4Model(args, config) + print("{:<40} {:<10}".format("Model type: ", "DeepSeekV4Model")) + elif config.is_hybrid_linear: model = HybridModel(args, config) print("{:<40} {:<10}".format("Model type: ", "HybridModel")) else: @@ -154,5 +178,68 @@ def main(args): parser.add_argument( "--decode-only", action="store_true", help="Only simulate decoding" ) + parser.add_argument( + "--dsv4-bench-data-dir", + type=str, + default=None, + help=("DeepSeek-V4 benchmark root. Defaults to bench_data/dsv4."), + ) + parser.add_argument( + "--dsv4-attn-backend", + type=str, + default="dsv4", + help="Attention backend key recorded in the V4 CSVs.", + ) + parser.add_argument( + "--dsv4-moe-backend", + type=str, + default="flashinfer_mxfp4", + help="Resolved MoE runner key recorded in the V4 CSVs.", + ) + parser.add_argument( + "--dsv4-attention-weight-dtype", + choices=["fp8", "bf16"], + default=None, + help="V4 attention weight dtype. Defaults to the model config.", + ) + parser.add_argument( + "--dsv4-expert-weight-dtype", + choices=["fp4", "fp8", "bf16"], + default=None, + help="V4 expert weight dtype. Defaults to the model config.", + ) + parser.add_argument( + "--dsv4-kv-dtype", + choices=["fp8_e4m3", "fp8", "bf16", "bfloat16"], + default="fp8_e4m3", + help="KV-cache dtype key recorded in the V4 attention CSV.", + ) + parser.add_argument( + "--dsv4-compress-state-dtype", + choices=["fp32", "bf16"], + default="fp32", + help="Compression-state dtype key recorded in the V4 attention CSV.", + ) + parser.add_argument( + "--dsv4-mhc-mode", + type=str, + default="fused_post_pre", + help="mHC execution-mode key recorded in the V4 CSV.", + ) + parser.add_argument( + "--dsv4-prefill-batch-size", + type=int, + default=None, + help=( + "Prefill request batch size. Defaults to " + "ceil(max-prefill-tokens / target-isl)." + ), + ) + parser.add_argument( + "--dsv4-decode-past-len", + type=int, + default=None, + help="Decode past length used for exact V4 CSV lookup.", + ) args = parser.parse_args() main(args) diff --git a/models/deepseek_v4_model.py b/models/deepseek_v4_model.py new file mode 100644 index 0000000..4946d98 --- /dev/null +++ b/models/deepseek_v4_model.py @@ -0,0 +1,236 @@ +"""Table-driven end-to-end simulator for DeepSeek-V4.""" + +import math + +from comm.comm import Comm +from config.model_config import get_deepseek_v4_architecture_fingerprint +from hardware.gpu import gpu_map +from kernel_sim.dsv4 import ( + DSV4BenchmarkData, + canonical_dsv4_attention_backend, + canonical_dsv4_kv_dtype, + canonical_dsv4_moe_key, +) + + +class DeepSeekV4Model: + """Sum exact measured C4/C128 attention, mHC and MoE layer latencies.""" + + def __init__(self, args, config): + self.args = args + self.config = config + self.device_type = self._arg("device_type", "H20") + self.gpu = gpu_map[self.device_type] + self.world_size = self._arg("world_size", 1) + self.tp_size = self._arg("tp_size", 1) + self.num_nodes = self._arg("num_nodes", 1) + if ( + self.tp_size <= 0 + or self.world_size < self.tp_size + or self.world_size % self.tp_size + ): + raise ValueError("world_size must be a positive multiple of tp_size") + if self.config.num_attention_heads % self.tp_size: + raise ValueError( + "DeepSeek-V4 num_attention_heads must be divisible by tp_size" + ) + + self.attention_weight_dtype = ( + self._arg("dsv4_attention_weight_dtype", None) + or config.attention_weight_dtype + ) + self.expert_weight_dtype = ( + self._arg("dsv4_expert_weight_dtype", None) or config.expert_dtype + ) + self.kv_dtype = canonical_dsv4_kv_dtype(self._arg("dsv4_kv_dtype", "fp8_e4m3")) + self.compress_state_dtype = self._arg("dsv4_compress_state_dtype", "fp32") + self.attn_backend = canonical_dsv4_attention_backend( + self._arg("dsv4_attn_backend", "dsv4") + ) + self.moe_backend, self.expert_weight_dtype = canonical_dsv4_moe_key( + self._arg("dsv4_moe_backend", "flashinfer_mxfp4"), + self.expert_weight_dtype, + ) + self.mhc_mode = self._arg("dsv4_mhc_mode", "fused_post_pre") + self.benchmarks = DSV4BenchmarkData( + self._arg("dsv4_bench_data_dir", None), + architecture_fingerprint=get_deepseek_v4_architecture_fingerprint(config), + ) + + self.target_isl = self._arg("target_isl", 4096) + self.target_osl = self._arg("target_osl", 2048) + self.max_prefill_tokens = self._arg("max_prefill_tokens", 4096) + self.decode_past_len = self._arg("dsv4_decode_past_len", None) + if self.decode_past_len is None: + self.decode_past_len = int(self.target_isl + self.target_osl / 2) + decode_bs = self._arg("decode_bs", None) + if decode_bs is None: + decode_bs = math.ceil( + self._arg("target_tgs", 2560) * self._arg("target_tpot", 50) / 1000 + ) + self.target_bs = decode_bs + + def _arg(self, name, default): + return getattr(self.args, name, default) + + def _prefill_batch_size(self): + explicit = self._arg("dsv4_prefill_batch_size", None) + if explicit is not None: + return explicit + return math.ceil(self.max_prefill_tokens / self.target_isl) + + def _comm_time_us(self, phase, num_tokens): + comm = Comm( + self.config, + self.gpu, + self.world_size, + self.num_nodes, + self._arg("enable_deepep", False), + ) + if self.tp_size > 1: + # One reduction follows attention and another follows MoE. + per_layer = 2 * comm.tp_all_reduce(num_tokens, self.tp_size) + elif phase == "prefill": + before_moe, after_moe = comm.prefill_comm(num_tokens) + per_layer = before_moe + after_moe + else: + before_moe, after_moe = comm.decode_comm(num_tokens) + per_layer = before_moe + after_moe + return per_layer * self.config.num_hidden_layers * 1e6 + + def _phase_latency_us(self, phase, batch_size, q_len, past_len): + attention = {} + for ratio, layer_count in self.config.attention_ratio_counts.items(): + one_layer_us = self.benchmarks.attention_latency_us( + phase=phase, + device_type=self.device_type, + tp_size=self.tp_size, + backend=self.attn_backend, + attention_weight_dtype=self.attention_weight_dtype, + kv_dtype=self.kv_dtype, + compress_state_dtype=self.compress_state_dtype, + ratio=ratio, + batch_size=batch_size, + q_len=q_len, + past_len=past_len, + ) + attention[ratio] = layer_count * one_layer_us + + mhc_us = self.config.num_hidden_layers * self.benchmarks.mhc_latency_us( + phase=phase, + device_type=self.device_type, + tp_size=self.tp_size, + backend=self.attn_backend, + attention_weight_dtype=self.attention_weight_dtype, + batch_size=batch_size, + q_len=q_len, + mode=self.mhc_mode, + ) + moe_us = self.config.num_hidden_layers * self.benchmarks.moe_latency_us( + phase=phase, + device_type=self.device_type, + tp_size=self.tp_size, + world_size=self.world_size, + num_nodes=self.num_nodes, + backend=self.moe_backend, + expert_weight_dtype=self.expert_weight_dtype, + batch_size=batch_size, + q_len=q_len, + ) + return { + "attention": attention, + "mhc": mhc_us, + "moe": moe_us, + "comm": self._comm_time_us(phase, batch_size * q_len), + } + + @staticmethod + def _print_phase_latency(name, timings): + for ratio, latency_us in sorted(timings["attention"].items()): + print("{:<40} {:<10.2f}".format(f"C{ratio} attention (us):", latency_us)) + print("{:<40} {:<10.2f}".format("mHC (us):", timings["mhc"])) + print("{:<40} {:<10.2f}".format("MoE (us):", timings["moe"])) + print("{:<40} {:<10.2f}".format("Communication (us):", timings["comm"])) + total = ( + sum(timings["attention"].values()) + + timings["mhc"] + + timings["moe"] + + timings["comm"] + ) + print("{:<40} {:<10.2f}".format(f"{name} total (us):", total)) + return total + + def print_weights_info(self): + print("{s:{c}^{n}}".format(s="Model Weights", n=50, c="-")) + print( + "{:<40} {:<10}".format( + "V4 attention weight dtype:", self.attention_weight_dtype + ) + ) + print( + "{:<40} {:<10}".format("V4 expert weight dtype:", self.expert_weight_dtype) + ) + print("{:<40} {:<10}".format("V4 MoE backend:", self.moe_backend)) + + def print_kvcache_info(self): + print("{s:{c}^{n}}".format(s="KV Cache", n=50, c="-")) + print("{:<40} {:<10}".format("V4 KV dtype:", self.kv_dtype)) + print("{:<40} {:<10}".format("Input seq len:", self.target_isl)) + print("{:<40} {:<10}".format("Output seq len:", self.target_osl)) + print("{:<40} {:<10}".format("Decode batch size:", self.target_bs)) + + def print_flops_info(self): + print("{s:{c}^{n}}".format(s="V4 Kernel Data", n=50, c="-")) + groups = ", ".join( + f"C{ratio} x {layer_count}" + for ratio, layer_count in self.config.attention_ratio_counts.items() + ) + print("{:<40} {:<10}".format("Attention layer groups:", groups)) + print("{:<40} {:<10}".format("mHC mode:", self.mhc_mode)) + print( + "Attention, mHC and MoE use exact measured rows; no generic " + "MLA/MHA MFU fallback is applied." + ) + + def _ensure_supported_mode(self): + if self._arg("enable_tbo", False): + raise ValueError( + "--enable-tbo requires overlapping V4 component measurements" + ) + + def prefill(self): + self._ensure_supported_mode() + print("{s:{c}^{n}}".format(s="Prefilling", n=50, c="-")) + batch_size = self._prefill_batch_size() + timings = self._phase_latency_us("prefill", batch_size, self.target_isl, 0) + total_us = self._print_phase_latency("Prefill", timings) + ttft_ms = total_us / 1000 + 30 + tokens = batch_size * self.target_isl + print("{:<40} {:<10.2f}".format("TTFT (ms):", ttft_ms)) + print( + "{:<40} {:<10.0f}".format( + "Throughput (TGS:tok/GPU/s):", + tokens / self.tp_size / (ttft_ms / 1000), + ) + ) + + def decoding(self): + self._ensure_supported_mode() + print("{s:{c}^{n}}".format(s="Decoding", n=50, c="-")) + timings = self._phase_latency_us( + "decode", self.target_bs, 1, self.decode_past_len + ) + total_us = self._print_phase_latency("Decode", timings) + overhead_ms = self._arg("decode_scheduler_overhead_ms", None) + if overhead_ms is None: + overhead_ms = 5 + tpot_ms = total_us / 1000 + overhead_ms + print("{:<40} {:<10.2f}".format("TPOT (ms):", tpot_ms)) + print( + "{:<40} {:<10.0f}".format( + "Throughput (TGS:tok/GPU/s):", + self.target_bs / self.tp_size / (tpot_ms / 1000), + ) + ) + if tpot_ms > self._arg("target_tpot", 50): + print("!Error: TPOT > SLO, need smaller GFLOPs to speedup") diff --git a/tests/fixtures/deepseek_v4/bench_data/dsv4/decode/h20/attention.csv b/tests/fixtures/deepseek_v4/bench_data/dsv4/decode/h20/attention.csv new file mode 100644 index 0000000..8272acc --- /dev/null +++ b/tests/fixtures/deepseek_v4/bench_data/dsv4/decode/h20/attention.csv @@ -0,0 +1,3 @@ +architecture_fingerprint,tp_size,backend,attention_weight_dtype,kv_dtype,compress_state_dtype,ratio,batch_size,q_len,past_len,latency_us +2a60524de4a88a10,16,dsv4,fp8,fp8_e4m3,fp32,128,4,1,12,10 +2a60524de4a88a10,16,dsv4,fp8,fp8_e4m3,fp32,4,4,1,12,20 diff --git a/tests/fixtures/deepseek_v4/bench_data/dsv4/decode/h20/mhc.csv b/tests/fixtures/deepseek_v4/bench_data/dsv4/decode/h20/mhc.csv new file mode 100644 index 0000000..0170e74 --- /dev/null +++ b/tests/fixtures/deepseek_v4/bench_data/dsv4/decode/h20/mhc.csv @@ -0,0 +1,2 @@ +architecture_fingerprint,tp_size,backend,attention_weight_dtype,batch_size,q_len,mode,latency_us +2a60524de4a88a10,16,dsv4,fp8,4,1,fused_post_pre,4 diff --git a/tests/fixtures/deepseek_v4/bench_data/dsv4/decode/h20/moe.csv b/tests/fixtures/deepseek_v4/bench_data/dsv4/decode/h20/moe.csv new file mode 100644 index 0000000..483f029 --- /dev/null +++ b/tests/fixtures/deepseek_v4/bench_data/dsv4/decode/h20/moe.csv @@ -0,0 +1,2 @@ +architecture_fingerprint,tp_size,world_size,num_nodes,backend,expert_weight_dtype,batch_size,q_len,latency_us +2a60524de4a88a10,16,16,2,flashinfer_mxfp4,fp4,4,1,5 diff --git a/tests/fixtures/deepseek_v4/bench_data/dsv4/prefill/h20/attention.csv b/tests/fixtures/deepseek_v4/bench_data/dsv4/prefill/h20/attention.csv new file mode 100644 index 0000000..ff2c778 --- /dev/null +++ b/tests/fixtures/deepseek_v4/bench_data/dsv4/prefill/h20/attention.csv @@ -0,0 +1,3 @@ +architecture_fingerprint,tp_size,backend,attention_weight_dtype,kv_dtype,compress_state_dtype,ratio,batch_size,q_len,past_len,latency_us +2a60524de4a88a10,16,dsv4,fp8,fp8_e4m3,fp32,128,1,8,0,10 +2a60524de4a88a10,16,dsv4,fp8,fp8_e4m3,fp32,4,1,8,0,20 diff --git a/tests/fixtures/deepseek_v4/bench_data/dsv4/prefill/h20/mhc.csv b/tests/fixtures/deepseek_v4/bench_data/dsv4/prefill/h20/mhc.csv new file mode 100644 index 0000000..84341f2 --- /dev/null +++ b/tests/fixtures/deepseek_v4/bench_data/dsv4/prefill/h20/mhc.csv @@ -0,0 +1,2 @@ +architecture_fingerprint,tp_size,backend,attention_weight_dtype,batch_size,q_len,mode,latency_us +2a60524de4a88a10,16,dsv4,fp8,1,8,fused_post_pre,4 diff --git a/tests/fixtures/deepseek_v4/bench_data/dsv4/prefill/h20/moe.csv b/tests/fixtures/deepseek_v4/bench_data/dsv4/prefill/h20/moe.csv new file mode 100644 index 0000000..4743f49 --- /dev/null +++ b/tests/fixtures/deepseek_v4/bench_data/dsv4/prefill/h20/moe.csv @@ -0,0 +1,2 @@ +architecture_fingerprint,tp_size,world_size,num_nodes,backend,expert_weight_dtype,batch_size,q_len,latency_us +2a60524de4a88a10,16,16,2,flashinfer_mxfp4,fp4,1,8,5 diff --git a/tests/fixtures/deepseek_v4/config.json b/tests/fixtures/deepseek_v4/config.json new file mode 100644 index 0000000..0b37371 --- /dev/null +++ b/tests/fixtures/deepseek_v4/config.json @@ -0,0 +1,23 @@ +{ + "model_type": "deepseek_v4", + "hidden_size": 7168, + "num_hidden_layers": 3, + "head_dim": 512, + "num_attention_heads": 128, + "num_key_value_heads": 1, + "q_lora_rank": 1536, + "o_lora_rank": 1024, + "qk_rope_head_dim": 64, + "o_groups": 16, + "index_topk": 1024, + "sliding_window": 128, + "hc_mult": 4, + "hc_sinkhorn_iters": 20, + "expert_dtype": "fp4", + "quantization_config": {"fmt": "e4m3"}, + "compress_ratios": [128, 4, 128, 0], + "n_routed_experts": 384, + "n_shared_experts": 1, + "num_experts_per_tok": 6, + "moe_intermediate_size": 3072 +} diff --git a/tests/test_deepseek_v4_simulator.py b/tests/test_deepseek_v4_simulator.py new file mode 100644 index 0000000..c21bedd --- /dev/null +++ b/tests/test_deepseek_v4_simulator.py @@ -0,0 +1,111 @@ +import unittest +from pathlib import Path +from types import SimpleNamespace + +from config.model_config import ModelConfig +from kernel_sim.dsv4 import DSV4BenchmarkData, MissingDSV4BenchmarkData +from models.deepseek_v4_model import DeepSeekV4Model + +FIXTURE_ROOT = Path(__file__).parent / "fixtures" / "deepseek_v4" + + +class DeepSeekV4SimulatorTest(unittest.TestCase): + def setUp(self): + self.config = ModelConfig(FIXTURE_ROOT / "config.json") + self.data_root = FIXTURE_ROOT / "bench_data" / "dsv4" + + def test_config_groups_base_layers_and_excludes_mtp(self): + self.assertTrue(self.config.is_deepseek_v4) + self.assertEqual(self.config.attn_type, "DSV4") + self.assertEqual(self.config.nextn_compress_ratios, [0]) + self.assertEqual(self.config.attention_ratio_counts, {128: 2, 4: 1}) + self.assertEqual( + self.config.dsv4_architecture_fingerprint, + "2a60524de4a88a10", + ) + + def test_lookup_requires_an_exact_measured_shape(self): + data = DSV4BenchmarkData( + self.data_root, + architecture_fingerprint=(self.config.dsv4_architecture_fingerprint), + ) + self.assertEqual( + data.attention_latency_us( + phase="decode", + device_type="H20", + tp_size=16, + backend="dsv4", + attention_weight_dtype="fp8", + kv_dtype="fp8_e4m3", + compress_state_dtype="fp32", + ratio=4, + batch_size=4, + q_len=1, + past_len=12, + ), + 20, + ) + with self.assertRaises(MissingDSV4BenchmarkData): + data.attention_latency_us( + phase="decode", + device_type="H20", + tp_size=16, + backend="dsv4", + attention_weight_dtype="fp8", + kv_dtype="fp8_e4m3", + compress_state_dtype="fp32", + ratio=4, + batch_size=5, + q_len=1, + past_len=12, + ) + + def test_packaged_h20_tp8_measurement_is_readable(self): + data = DSV4BenchmarkData( + Path(__file__).parents[1] / "bench_data" / "dsv4", + architecture_fingerprint="dc8a15770f7f8647", + ) + self.assertAlmostEqual( + data.attention_latency_us( + phase="prefill", + device_type="H20", + tp_size=8, + backend="dsv4", + attention_weight_dtype="fp8", + kv_dtype="fp8_e4m3", + compress_state_dtype="fp32", + ratio=128, + batch_size=1, + q_len=4096, + past_len=0, + ), + 3988.013406, + ) + + def test_model_sums_ratio_groups_mhc_and_moe(self): + args = SimpleNamespace( + device_type="H20", + world_size=16, + tp_size=16, + num_nodes=2, + dsv4_bench_data_dir=str(self.data_root), + target_isl=8, + target_osl=8, + max_prefill_tokens=8, + decode_bs=4, + dsv4_decode_past_len=12, + target_tpot=50, + enable_deepep=False, + enable_tbo=False, + ) + model = DeepSeekV4Model(args, self.config) + model._comm_time_us = lambda phase, tokens: 7 + timings = model._phase_latency_us("decode", 4, 1, 12) + self.assertEqual(timings["attention"], {128: 20, 4: 20}) + self.assertEqual(timings["mhc"], 12) + self.assertEqual(timings["moe"], 15) + self.assertEqual(timings["comm"], 7) + + +if __name__ == "__main__": + unittest.main()