|
| 1 | +# SPDX-License-Identifier: Apache-2.0 |
| 2 | +# DeepSpeed Team |
| 3 | +"""Reproducible OPSD continuous-batching benchmark. |
| 4 | +
|
| 5 | +The benchmark compares independent Hugging Face generation, capacity-bounded |
| 6 | +static batches, and :meth:`HybridEngineRollout.generate_continuous`. Prompts |
| 7 | +are deterministic token IDs; tokenization and model loading are outside the |
| 8 | +timed region. |
| 9 | +""" |
| 10 | + |
| 11 | +import argparse |
| 12 | +import json |
| 13 | +import math |
| 14 | +import statistics |
| 15 | +import time |
| 16 | +from pathlib import Path |
| 17 | + |
| 18 | + |
| 19 | +def percentile(values, quantile): |
| 20 | + """Return the OPSD percentile (nearest-rank, ceil-based) used elsewhere.""" |
| 21 | + if not values: |
| 22 | + raise ValueError("values must not be empty") |
| 23 | + ordered = sorted(values) |
| 24 | + index = max(0, math.ceil(len(ordered) * quantile) - 1) |
| 25 | + return ordered[index] |
| 26 | + |
| 27 | + |
| 28 | +def summarize_latencies(latencies): |
| 29 | + """Summarize measured milliseconds with the existing OPSD percentile rule.""" |
| 30 | + if not latencies: |
| 31 | + raise ValueError("latencies must not be empty") |
| 32 | + return { |
| 33 | + "mean": statistics.mean(latencies), |
| 34 | + "p50": percentile(latencies, 0.50), |
| 35 | + "p95": percentile(latencies, 0.95), |
| 36 | + } |
| 37 | + |
| 38 | + |
| 39 | +def group_request_indices(response_lengths, max_batch_size): |
| 40 | + """Group requests in input order into capacity-sized static batches.""" |
| 41 | + if max_batch_size <= 0: |
| 42 | + raise ValueError("max_batch_size must be positive") |
| 43 | + return [tuple(range(start, min(start + max_batch_size, len(response_lengths)))) |
| 44 | + for start in range(0, len(response_lengths), max_batch_size)] |
| 45 | + |
| 46 | + |
| 47 | +def static_computed_tokens(response_lengths, max_batch_size): |
| 48 | + """Count tokens computed by static batching, including short-request extras.""" |
| 49 | + return sum(len(group) * max(response_lengths[index] for index in group) |
| 50 | + for group in group_request_indices(response_lengths, max_batch_size)) |
| 51 | + |
| 52 | + |
| 53 | +def validate_args(args): |
| 54 | + if args.dtype not in ("fp16", "bf16"): |
| 55 | + raise ValueError("dtype must be fp16 or bf16") |
| 56 | + if args.prompt_length <= 0 or args.max_batch_size <= 0: |
| 57 | + raise ValueError("prompt-length and max-batch-size must be positive") |
| 58 | + if not args.response_lengths or any(length <= 0 for length in args.response_lengths): |
| 59 | + raise ValueError("response-lengths must contain positive lengths") |
| 60 | + if args.warmup < 0 or args.iterations <= 0: |
| 61 | + raise ValueError("warmup must be non-negative and iterations must be positive") |
| 62 | + if args.temperature != 0.0: |
| 63 | + raise ValueError("this benchmark currently supports greedy generation only (temperature=0)") |
| 64 | + if args.seed < 0: |
| 65 | + raise ValueError("seed must be non-negative") |
| 66 | + |
| 67 | + |
| 68 | +def _build_parser(): |
| 69 | + parser = argparse.ArgumentParser(description="Benchmark OPSD continuous batching") |
| 70 | + parser.add_argument("--model", default="facebook/opt-125m") |
| 71 | + parser.add_argument("--dtype", choices=("fp16", "bf16"), default="fp16") |
| 72 | + parser.add_argument("--prompt-length", type=int, default=512) |
| 73 | + parser.add_argument("--response-lengths", type=int, nargs="+", default=[32, 64, 96, 128]) |
| 74 | + parser.add_argument("--max-batch-size", type=int, default=2) |
| 75 | + parser.add_argument("--warmup", type=int, default=1) |
| 76 | + parser.add_argument("--iterations", type=int, default=3) |
| 77 | + parser.add_argument("--temperature", type=float, default=0.0) |
| 78 | + parser.add_argument("--seed", type=int, default=1234) |
| 79 | + parser.add_argument("--output", default="opsd_continuous_batching.json") |
| 80 | + return parser |
| 81 | + |
| 82 | + |
| 83 | +def _synthetic_prompts(vocab_size, pad_token_id, prompt_length, count, device, seed=0): |
| 84 | + """Build deterministic, equal-width prompts without pad IDs.""" |
| 85 | + if vocab_size <= 1: |
| 86 | + raise ValueError("model vocabulary must contain at least two tokens") |
| 87 | + available = [token for token in range(vocab_size) if token != pad_token_id] |
| 88 | + if not available: |
| 89 | + raise ValueError("model vocabulary has no token distinct from pad_token_id") |
| 90 | + values = [available[(seed + offset * prompt_length + position) % len(available)] |
| 91 | + for offset in range(count) for position in range(prompt_length)] |
| 92 | + import torch |
| 93 | + return [torch.tensor(values[offset * prompt_length:(offset + 1) * prompt_length], |
| 94 | + dtype=torch.long, device=device).unsqueeze(0) |
| 95 | + for offset in range(count)] |
| 96 | + |
| 97 | + |
| 98 | +def _sampling_config(length, temperature): |
| 99 | + from deepspeed.runtime.rollout.base import SamplingConfig |
| 100 | + return SamplingConfig(max_new_tokens=length, temperature=temperature, top_p=1.0, n_samples_per_prompt=1) |
| 101 | + |
| 102 | + |
| 103 | +def _request(prompt_ids): |
| 104 | + from deepspeed.runtime.rollout.base import RolloutRequest |
| 105 | + import torch |
| 106 | + return RolloutRequest(prompt_ids=prompt_ids, prompt_attention_mask=torch.ones_like(prompt_ids)) |
| 107 | + |
| 108 | + |
| 109 | +def _sync(): |
| 110 | + import torch |
| 111 | + torch.cuda.synchronize() |
| 112 | + |
| 113 | + |
| 114 | +def _reset_peak_memory(): |
| 115 | + import torch |
| 116 | + torch.cuda.reset_peak_memory_stats() |
| 117 | + |
| 118 | + |
| 119 | +def _peak_memory_mb(): |
| 120 | + import torch |
| 121 | + return torch.cuda.max_memory_allocated() / (1024**2) |
| 122 | + |
| 123 | + |
| 124 | +def _response(output_ids, prompt_length, response_length): |
| 125 | + return output_ids[0, prompt_length:prompt_length + response_length].detach().cpu() |
| 126 | + |
| 127 | + |
| 128 | +def _assert_matches(reference, candidate, mode, request_index): |
| 129 | + if len(reference) != len(candidate) or not reference.equal(candidate): |
| 130 | + raise AssertionError(f"{mode} request {request_index} response tokens differ from sequential_eager") |
| 131 | + |
| 132 | + |
| 133 | +def _run_sequential(generate, model_config, requests, response_lengths, args, prompt_length): |
| 134 | + latencies = [] |
| 135 | + reference_responses = [] |
| 136 | + pad_token_id = getattr(model_config, "pad_token_id", None) |
| 137 | + for _ in range(args.warmup): |
| 138 | + for request, length in zip(requests, response_lengths): |
| 139 | + generate(request.prompt_ids, attention_mask=request.prompt_attention_mask, |
| 140 | + max_new_tokens=length, min_new_tokens=length, eos_token_id=None, |
| 141 | + do_sample=False, pad_token_id=pad_token_id) |
| 142 | + _reset_peak_memory() |
| 143 | + for _ in range(args.iterations): |
| 144 | + _sync() |
| 145 | + start = time.perf_counter() |
| 146 | + outputs = [] |
| 147 | + for request, length in zip(requests, response_lengths): |
| 148 | + output = generate(request.prompt_ids, attention_mask=request.prompt_attention_mask, |
| 149 | + max_new_tokens=length, min_new_tokens=length, eos_token_id=None, |
| 150 | + do_sample=False, pad_token_id=pad_token_id) |
| 151 | + outputs.append(_response(output, prompt_length, length)) |
| 152 | + _sync() |
| 153 | + latencies.append((time.perf_counter() - start) * 1000.0) |
| 154 | + if not reference_responses: |
| 155 | + reference_responses = outputs |
| 156 | + else: |
| 157 | + for index, response in enumerate(outputs): |
| 158 | + _assert_matches(reference_responses[index], response, "sequential_eager", index) |
| 159 | + return latencies, reference_responses, _peak_memory_mb() |
| 160 | + |
| 161 | + |
| 162 | +def _run_static(generate, model_config, requests, response_lengths, args, prompt_length, reference_responses): |
| 163 | + latencies = [] |
| 164 | + groups = group_request_indices(response_lengths, args.max_batch_size) |
| 165 | + pad_token_id = getattr(model_config, "pad_token_id", None) |
| 166 | + |
| 167 | + def invoke(): |
| 168 | + outputs = [None] * len(requests) |
| 169 | + for group in groups: |
| 170 | + max_length = max(response_lengths[index] for index in group) |
| 171 | + prompt_ids = __import__("torch").cat([requests[index].prompt_ids for index in group], dim=0) |
| 172 | + attention = __import__("torch").cat([requests[index].prompt_attention_mask for index in group], dim=0) |
| 173 | + generated = generate(prompt_ids, attention_mask=attention, max_new_tokens=max_length, |
| 174 | + min_new_tokens=max_length, eos_token_id=None, do_sample=False, |
| 175 | + pad_token_id=pad_token_id) |
| 176 | + for row, index in enumerate(group): |
| 177 | + outputs[index] = generated[row, prompt_length:prompt_length + response_lengths[index]].detach().cpu() |
| 178 | + return outputs |
| 179 | + |
| 180 | + for _ in range(args.warmup): |
| 181 | + invoke() |
| 182 | + _reset_peak_memory() |
| 183 | + for _ in range(args.iterations): |
| 184 | + _sync() |
| 185 | + start = time.perf_counter() |
| 186 | + outputs = invoke() |
| 187 | + _sync() |
| 188 | + latencies.append((time.perf_counter() - start) * 1000.0) |
| 189 | + for index, response in enumerate(outputs): |
| 190 | + _assert_matches(reference_responses[index], response, "static_batch", index) |
| 191 | + return latencies, _peak_memory_mb() |
| 192 | + |
| 193 | + |
| 194 | +def _run_continuous(rollout, requests, response_lengths, args, prompt_length, reference_responses): |
| 195 | + latencies = [] |
| 196 | + configs = [_sampling_config(length, args.temperature) for length in response_lengths] |
| 197 | + for _ in range(args.warmup): |
| 198 | + rollout.generate_continuous(requests, configs, max_batch_size=args.max_batch_size) |
| 199 | + _reset_peak_memory() |
| 200 | + for _ in range(args.iterations): |
| 201 | + _sync() |
| 202 | + start = time.perf_counter() |
| 203 | + outputs = rollout.generate_continuous(requests, configs, max_batch_size=args.max_batch_size) |
| 204 | + _sync() |
| 205 | + latencies.append((time.perf_counter() - start) * 1000.0) |
| 206 | + for index, (output, length) in enumerate(zip(outputs, response_lengths)): |
| 207 | + response = output.input_ids[0, prompt_length:prompt_length + length].detach().cpu() |
| 208 | + _assert_matches(reference_responses[index], response, "continuous_batch", index) |
| 209 | + return latencies, _peak_memory_mb() |
| 210 | + |
| 211 | + |
| 212 | +def _mode_result(latencies, useful_tokens, computed_tokens, peak_memory): |
| 213 | + summary = summarize_latencies(latencies) |
| 214 | + mean_latency = summary["mean"] |
| 215 | + return { |
| 216 | + "latency_ms": summary, |
| 217 | + "useful_tokens": useful_tokens, |
| 218 | + "computed_tokens": computed_tokens, |
| 219 | + "useful_tokens_per_second": useful_tokens / (mean_latency / 1000.0), |
| 220 | + "peak_memory_mb": peak_memory, |
| 221 | + } |
| 222 | + |
| 223 | + |
| 224 | +def _change_percent(value, baseline): |
| 225 | + return (value - baseline) / baseline * 100.0 |
| 226 | + |
| 227 | + |
| 228 | +def _build_result(args, environment, mode_results, useful_tokens): |
| 229 | + sequential = mode_results["sequential_eager"] |
| 230 | + continuous = mode_results["continuous_batch"] |
| 231 | + static = mode_results["static_batch"] |
| 232 | + return { |
| 233 | + "environment": environment, |
| 234 | + "config": {"model": args.model, "dtype": args.dtype, "prompt_length": args.prompt_length, |
| 235 | + "response_lengths": args.response_lengths, "max_batch_size": args.max_batch_size, |
| 236 | + "warmup": args.warmup, "iterations": args.iterations, |
| 237 | + "temperature": args.temperature, "seed": args.seed}, |
| 238 | + "results": mode_results, |
| 239 | + "comparisons": { |
| 240 | + "continuous_vs_sequential": { |
| 241 | + "latency_change_percent": _change_percent(continuous["latency_ms"]["mean"], |
| 242 | + sequential["latency_ms"]["mean"]), |
| 243 | + "useful_throughput_change_percent": _change_percent(continuous["useful_tokens_per_second"], |
| 244 | + sequential["useful_tokens_per_second"]), |
| 245 | + }, |
| 246 | + "continuous_vs_static": { |
| 247 | + "latency_change_percent": _change_percent(continuous["latency_ms"]["mean"], |
| 248 | + static["latency_ms"]["mean"]), |
| 249 | + "useful_throughput_change_percent": _change_percent(continuous["useful_tokens_per_second"], |
| 250 | + static["useful_tokens_per_second"]), |
| 251 | + "avoided_decode_tokens": static["computed_tokens"] - continuous["computed_tokens"], |
| 252 | + }, |
| 253 | + }, |
| 254 | + } |
| 255 | + |
| 256 | + |
| 257 | +def run(args): |
| 258 | + validate_args(args) |
| 259 | + import os |
| 260 | + import torch |
| 261 | + from transformers import AutoModelForCausalLM, AutoTokenizer |
| 262 | + import deepspeed |
| 263 | + from deepspeed.runtime.rollout.hybrid_engine_rollout import HybridEngineRollout, HybridEngineRolloutConfig |
| 264 | + |
| 265 | + if not torch.cuda.is_available(): |
| 266 | + raise RuntimeError("continuous batching benchmark requires CUDA") |
| 267 | + if int(os.getenv("WORLD_SIZE", "1")) != 1: |
| 268 | + raise RuntimeError("this benchmark supports exactly one GPU process") |
| 269 | + torch.manual_seed(args.seed) |
| 270 | + device = torch.device("cuda", int(os.getenv("LOCAL_RANK", "0"))) |
| 271 | + torch.cuda.set_device(device) |
| 272 | + dtype = torch.float16 if args.dtype == "fp16" else torch.bfloat16 |
| 273 | + |
| 274 | + tokenizer = AutoTokenizer.from_pretrained(args.model) |
| 275 | + pad_token_id = tokenizer.pad_token_id |
| 276 | + if pad_token_id is None: |
| 277 | + if tokenizer.eos_token_id is None: |
| 278 | + raise ValueError("tokenizer must define pad_token_id or eos_token_id") |
| 279 | + pad_token_id = tokenizer.eos_token_id |
| 280 | + tokenizer.pad_token = tokenizer.eos_token |
| 281 | + # The Core continuous scheduler otherwise treats model EOS as early completion. |
| 282 | + tokenizer.eos_token_id = None |
| 283 | + model = AutoModelForCausalLM.from_pretrained(args.model, torch_dtype=dtype, low_cpu_mem_usage=True).to(device) |
| 284 | + model.eval() |
| 285 | + model.config.pad_token_id = pad_token_id |
| 286 | + model.config.eos_token_id = None |
| 287 | + capacity = args.max_batch_size |
| 288 | + # Keep all three modes on the same unmodified HF forward/generate path. |
| 289 | + # Enabling DeepSpeed HybridEngine here replaces ``module.generate`` and |
| 290 | + # injects inference kernels whose batch-shape-dependent numerics can make |
| 291 | + # exact token comparison impossible. Core's rollout API only requires an |
| 292 | + # object exposing ``module`` for continuous generation. |
| 293 | + from types import SimpleNamespace |
| 294 | + engine = SimpleNamespace(module=model) |
| 295 | + rollout = HybridEngineRollout(engine, tokenizer, HybridEngineRolloutConfig()) |
| 296 | + prompts = _synthetic_prompts(model.config.vocab_size, pad_token_id, args.prompt_length, |
| 297 | + len(args.response_lengths), device, args.seed) |
| 298 | + requests = [_request(prompt) for prompt in prompts] |
| 299 | + |
| 300 | + hf_generate = model.generate |
| 301 | + sequential_latencies, references, sequential_peak = _run_sequential( |
| 302 | + hf_generate, engine.module.config, requests, args.response_lengths, args, args.prompt_length) |
| 303 | + static_latencies, static_peak = _run_static( |
| 304 | + hf_generate, engine.module.config, requests, args.response_lengths, args, args.prompt_length, references) |
| 305 | + continuous_latencies, continuous_peak = _run_continuous( |
| 306 | + rollout, requests, args.response_lengths, args, args.prompt_length, references) |
| 307 | + useful_tokens = sum(args.response_lengths) |
| 308 | + mode_results = { |
| 309 | + "sequential_eager": _mode_result(sequential_latencies, useful_tokens, useful_tokens, sequential_peak), |
| 310 | + "static_batch": _mode_result(static_latencies, useful_tokens, |
| 311 | + static_computed_tokens(args.response_lengths, capacity), static_peak), |
| 312 | + "continuous_batch": _mode_result(continuous_latencies, useful_tokens, useful_tokens, continuous_peak), |
| 313 | + } |
| 314 | + environment = {"torch": torch.__version__, "cuda": torch.version.cuda, |
| 315 | + "transformers": __import__("transformers").__version__, |
| 316 | + "deepspeed": deepspeed.__version__, "gpu": torch.cuda.get_device_name(device)} |
| 317 | + result = _build_result(args, environment, mode_results, useful_tokens) |
| 318 | + output_path = Path(args.output) |
| 319 | + output_path.parent.mkdir(parents=True, exist_ok=True) |
| 320 | + output_path.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") |
| 321 | + print(f"Wrote benchmark results to {output_path}") |
| 322 | + return result |
| 323 | + |
| 324 | + |
| 325 | +if __name__ == "__main__": |
| 326 | + run(_build_parser().parse_args()) |
0 commit comments