Skip to content

Commit d4e1a94

Browse files
committed
perf(opsd): add continuous batching benchmark
Signed-off-by: nathon-lee <leejianwoo@gmail.com>
1 parent 9311692 commit d4e1a94

3 files changed

Lines changed: 444 additions & 0 deletions

File tree

‎benchmarks/opsd/README.md‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,3 +51,61 @@ torchrun --nproc_per_node=1 \
5151
Use `--help` for the complete argument list. Model download, GPU memory, and
5252
the fused inference kernels supported by the selected model can limit which
5353
matrix shapes run successfully.
54+
55+
## Continuous batching benchmark
56+
57+
`benchmark_continuous_batching.py` measures the same synthetic request stream
58+
three ways: `sequential_eager` calls Hugging Face `model.generate()` once per
59+
request with that request's budget; `static_batch` groups requests in
60+
`--max-batch-size` capacity-sized batches and generates each group's longest
61+
budget (short requests therefore do extra decode work); and
62+
`continuous_batch` calls DeepSpeed Core's
63+
`HybridEngineRollout.generate_continuous(requests, sampling_configs,
64+
max_batch_size=...)`, retiring completed rows and admitting pending requests.
65+
66+
All modes use one loaded model, the same deterministic equal-width token-ID
67+
prompts, response budgets, seed, and capacity. Model loading is outside the
68+
timed region. Every measured iteration synchronizes CUDA before and after the
69+
work, resets peak memory statistics at the start of each mode, and checks every
70+
response token against the sequential eager baseline. Generation is greedy
71+
only (`--temperature 0`), uses `min_new_tokens == max_new_tokens`, and disables
72+
EOS (`eos_token_id=None`) so each request consumes its requested budget.
73+
74+
The JSON `environment` records torch/CUDA/Transformers/DeepSpeed versions and
75+
GPU. `config` records the CLI workload. Each `results` mode reports
76+
`latency_ms` (mean/p50/p95), useful and computed tokens, useful-token
77+
throughput, and peak allocated memory in MiB. Useful tokens are always the sum
78+
of response budgets; computed tokens are equal to useful tokens for sequential
79+
and continuous modes, while static batching uses
80+
`sum(group_size * group_max_response_length)`. `comparisons` reports percentage
81+
changes in mean latency and useful throughput plus static decode tokens avoided
82+
by continuous batching. p95 uses the same ceil-based nearest-rank percentile
83+
as the existing OPSD benchmark.
84+
85+
Current limitations are one GPU/process, greedy decoding, equal prompt width,
86+
and the legacy Hugging Face KV-cache path where the selected model does not
87+
expose the newer cache API. This benchmark does not copy or implement Core's
88+
continuous scheduler; load the DeepSpeed Core checkout containing that API via
89+
`PYTHONPATH`.
90+
91+
OPT-125M smoke test:
92+
93+
```bash
94+
PYTHONPATH=/workspace/DeepSpeed_woo:/workspace/DeepSpeedExamples_woo \
95+
torchrun --nproc_per_node=1 benchmarks/opsd/benchmark_continuous_batching.py \
96+
--model facebook/opt-125m --dtype fp16 --prompt-length 64 \
97+
--response-lengths 8 16 24 32 --max-batch-size 2 \
98+
--warmup 1 --iterations 3 --temperature 0 --seed 1234 \
99+
--output /workspace/results/opsd_continuous_batching_opt125m.json
100+
```
101+
102+
OPT-6.7B formal benchmark:
103+
104+
```bash
105+
PYTHONPATH=/workspace/DeepSpeed_woo:/workspace/DeepSpeedExamples_woo \
106+
torchrun --nproc_per_node=1 benchmarks/opsd/benchmark_continuous_batching.py \
107+
--model facebook/opt-6.7b --dtype fp16 --prompt-length 512 \
108+
--response-lengths 32 64 96 128 --max-batch-size 2 \
109+
--warmup 1 --iterations 3 --temperature 0 --seed 1234 \
110+
--output /workspace/results/opsd_continuous_batching_opt67b.json
111+
```
Lines changed: 326 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,326 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
# DeepSpeed Team
3+
"""Reproducible OPSD continuous-batching benchmark.
4+
5+
The benchmark compares independent Hugging Face generation, capacity-bounded
6+
static batches, and :meth:`HybridEngineRollout.generate_continuous`. Prompts
7+
are deterministic token IDs; tokenization and model loading are outside the
8+
timed region.
9+
"""
10+
11+
import argparse
12+
import json
13+
import math
14+
import statistics
15+
import time
16+
from pathlib import Path
17+
18+
19+
def percentile(values, quantile):
20+
"""Return the OPSD percentile (nearest-rank, ceil-based) used elsewhere."""
21+
if not values:
22+
raise ValueError("values must not be empty")
23+
ordered = sorted(values)
24+
index = max(0, math.ceil(len(ordered) * quantile) - 1)
25+
return ordered[index]
26+
27+
28+
def summarize_latencies(latencies):
29+
"""Summarize measured milliseconds with the existing OPSD percentile rule."""
30+
if not latencies:
31+
raise ValueError("latencies must not be empty")
32+
return {
33+
"mean": statistics.mean(latencies),
34+
"p50": percentile(latencies, 0.50),
35+
"p95": percentile(latencies, 0.95),
36+
}
37+
38+
39+
def group_request_indices(response_lengths, max_batch_size):
40+
"""Group requests in input order into capacity-sized static batches."""
41+
if max_batch_size <= 0:
42+
raise ValueError("max_batch_size must be positive")
43+
return [tuple(range(start, min(start + max_batch_size, len(response_lengths))))
44+
for start in range(0, len(response_lengths), max_batch_size)]
45+
46+
47+
def static_computed_tokens(response_lengths, max_batch_size):
48+
"""Count tokens computed by static batching, including short-request extras."""
49+
return sum(len(group) * max(response_lengths[index] for index in group)
50+
for group in group_request_indices(response_lengths, max_batch_size))
51+
52+
53+
def validate_args(args):
54+
if args.dtype not in ("fp16", "bf16"):
55+
raise ValueError("dtype must be fp16 or bf16")
56+
if args.prompt_length <= 0 or args.max_batch_size <= 0:
57+
raise ValueError("prompt-length and max-batch-size must be positive")
58+
if not args.response_lengths or any(length <= 0 for length in args.response_lengths):
59+
raise ValueError("response-lengths must contain positive lengths")
60+
if args.warmup < 0 or args.iterations <= 0:
61+
raise ValueError("warmup must be non-negative and iterations must be positive")
62+
if args.temperature != 0.0:
63+
raise ValueError("this benchmark currently supports greedy generation only (temperature=0)")
64+
if args.seed < 0:
65+
raise ValueError("seed must be non-negative")
66+
67+
68+
def _build_parser():
69+
parser = argparse.ArgumentParser(description="Benchmark OPSD continuous batching")
70+
parser.add_argument("--model", default="facebook/opt-125m")
71+
parser.add_argument("--dtype", choices=("fp16", "bf16"), default="fp16")
72+
parser.add_argument("--prompt-length", type=int, default=512)
73+
parser.add_argument("--response-lengths", type=int, nargs="+", default=[32, 64, 96, 128])
74+
parser.add_argument("--max-batch-size", type=int, default=2)
75+
parser.add_argument("--warmup", type=int, default=1)
76+
parser.add_argument("--iterations", type=int, default=3)
77+
parser.add_argument("--temperature", type=float, default=0.0)
78+
parser.add_argument("--seed", type=int, default=1234)
79+
parser.add_argument("--output", default="opsd_continuous_batching.json")
80+
return parser
81+
82+
83+
def _synthetic_prompts(vocab_size, pad_token_id, prompt_length, count, device, seed=0):
84+
"""Build deterministic, equal-width prompts without pad IDs."""
85+
if vocab_size <= 1:
86+
raise ValueError("model vocabulary must contain at least two tokens")
87+
available = [token for token in range(vocab_size) if token != pad_token_id]
88+
if not available:
89+
raise ValueError("model vocabulary has no token distinct from pad_token_id")
90+
values = [available[(seed + offset * prompt_length + position) % len(available)]
91+
for offset in range(count) for position in range(prompt_length)]
92+
import torch
93+
return [torch.tensor(values[offset * prompt_length:(offset + 1) * prompt_length],
94+
dtype=torch.long, device=device).unsqueeze(0)
95+
for offset in range(count)]
96+
97+
98+
def _sampling_config(length, temperature):
99+
from deepspeed.runtime.rollout.base import SamplingConfig
100+
return SamplingConfig(max_new_tokens=length, temperature=temperature, top_p=1.0, n_samples_per_prompt=1)
101+
102+
103+
def _request(prompt_ids):
104+
from deepspeed.runtime.rollout.base import RolloutRequest
105+
import torch
106+
return RolloutRequest(prompt_ids=prompt_ids, prompt_attention_mask=torch.ones_like(prompt_ids))
107+
108+
109+
def _sync():
110+
import torch
111+
torch.cuda.synchronize()
112+
113+
114+
def _reset_peak_memory():
115+
import torch
116+
torch.cuda.reset_peak_memory_stats()
117+
118+
119+
def _peak_memory_mb():
120+
import torch
121+
return torch.cuda.max_memory_allocated() / (1024**2)
122+
123+
124+
def _response(output_ids, prompt_length, response_length):
125+
return output_ids[0, prompt_length:prompt_length + response_length].detach().cpu()
126+
127+
128+
def _assert_matches(reference, candidate, mode, request_index):
129+
if len(reference) != len(candidate) or not reference.equal(candidate):
130+
raise AssertionError(f"{mode} request {request_index} response tokens differ from sequential_eager")
131+
132+
133+
def _run_sequential(generate, model_config, requests, response_lengths, args, prompt_length):
134+
latencies = []
135+
reference_responses = []
136+
pad_token_id = getattr(model_config, "pad_token_id", None)
137+
for _ in range(args.warmup):
138+
for request, length in zip(requests, response_lengths):
139+
generate(request.prompt_ids, attention_mask=request.prompt_attention_mask,
140+
max_new_tokens=length, min_new_tokens=length, eos_token_id=None,
141+
do_sample=False, pad_token_id=pad_token_id)
142+
_reset_peak_memory()
143+
for _ in range(args.iterations):
144+
_sync()
145+
start = time.perf_counter()
146+
outputs = []
147+
for request, length in zip(requests, response_lengths):
148+
output = generate(request.prompt_ids, attention_mask=request.prompt_attention_mask,
149+
max_new_tokens=length, min_new_tokens=length, eos_token_id=None,
150+
do_sample=False, pad_token_id=pad_token_id)
151+
outputs.append(_response(output, prompt_length, length))
152+
_sync()
153+
latencies.append((time.perf_counter() - start) * 1000.0)
154+
if not reference_responses:
155+
reference_responses = outputs
156+
else:
157+
for index, response in enumerate(outputs):
158+
_assert_matches(reference_responses[index], response, "sequential_eager", index)
159+
return latencies, reference_responses, _peak_memory_mb()
160+
161+
162+
def _run_static(generate, model_config, requests, response_lengths, args, prompt_length, reference_responses):
163+
latencies = []
164+
groups = group_request_indices(response_lengths, args.max_batch_size)
165+
pad_token_id = getattr(model_config, "pad_token_id", None)
166+
167+
def invoke():
168+
outputs = [None] * len(requests)
169+
for group in groups:
170+
max_length = max(response_lengths[index] for index in group)
171+
prompt_ids = __import__("torch").cat([requests[index].prompt_ids for index in group], dim=0)
172+
attention = __import__("torch").cat([requests[index].prompt_attention_mask for index in group], dim=0)
173+
generated = generate(prompt_ids, attention_mask=attention, max_new_tokens=max_length,
174+
min_new_tokens=max_length, eos_token_id=None, do_sample=False,
175+
pad_token_id=pad_token_id)
176+
for row, index in enumerate(group):
177+
outputs[index] = generated[row, prompt_length:prompt_length + response_lengths[index]].detach().cpu()
178+
return outputs
179+
180+
for _ in range(args.warmup):
181+
invoke()
182+
_reset_peak_memory()
183+
for _ in range(args.iterations):
184+
_sync()
185+
start = time.perf_counter()
186+
outputs = invoke()
187+
_sync()
188+
latencies.append((time.perf_counter() - start) * 1000.0)
189+
for index, response in enumerate(outputs):
190+
_assert_matches(reference_responses[index], response, "static_batch", index)
191+
return latencies, _peak_memory_mb()
192+
193+
194+
def _run_continuous(rollout, requests, response_lengths, args, prompt_length, reference_responses):
195+
latencies = []
196+
configs = [_sampling_config(length, args.temperature) for length in response_lengths]
197+
for _ in range(args.warmup):
198+
rollout.generate_continuous(requests, configs, max_batch_size=args.max_batch_size)
199+
_reset_peak_memory()
200+
for _ in range(args.iterations):
201+
_sync()
202+
start = time.perf_counter()
203+
outputs = rollout.generate_continuous(requests, configs, max_batch_size=args.max_batch_size)
204+
_sync()
205+
latencies.append((time.perf_counter() - start) * 1000.0)
206+
for index, (output, length) in enumerate(zip(outputs, response_lengths)):
207+
response = output.input_ids[0, prompt_length:prompt_length + length].detach().cpu()
208+
_assert_matches(reference_responses[index], response, "continuous_batch", index)
209+
return latencies, _peak_memory_mb()
210+
211+
212+
def _mode_result(latencies, useful_tokens, computed_tokens, peak_memory):
213+
summary = summarize_latencies(latencies)
214+
mean_latency = summary["mean"]
215+
return {
216+
"latency_ms": summary,
217+
"useful_tokens": useful_tokens,
218+
"computed_tokens": computed_tokens,
219+
"useful_tokens_per_second": useful_tokens / (mean_latency / 1000.0),
220+
"peak_memory_mb": peak_memory,
221+
}
222+
223+
224+
def _change_percent(value, baseline):
225+
return (value - baseline) / baseline * 100.0
226+
227+
228+
def _build_result(args, environment, mode_results, useful_tokens):
229+
sequential = mode_results["sequential_eager"]
230+
continuous = mode_results["continuous_batch"]
231+
static = mode_results["static_batch"]
232+
return {
233+
"environment": environment,
234+
"config": {"model": args.model, "dtype": args.dtype, "prompt_length": args.prompt_length,
235+
"response_lengths": args.response_lengths, "max_batch_size": args.max_batch_size,
236+
"warmup": args.warmup, "iterations": args.iterations,
237+
"temperature": args.temperature, "seed": args.seed},
238+
"results": mode_results,
239+
"comparisons": {
240+
"continuous_vs_sequential": {
241+
"latency_change_percent": _change_percent(continuous["latency_ms"]["mean"],
242+
sequential["latency_ms"]["mean"]),
243+
"useful_throughput_change_percent": _change_percent(continuous["useful_tokens_per_second"],
244+
sequential["useful_tokens_per_second"]),
245+
},
246+
"continuous_vs_static": {
247+
"latency_change_percent": _change_percent(continuous["latency_ms"]["mean"],
248+
static["latency_ms"]["mean"]),
249+
"useful_throughput_change_percent": _change_percent(continuous["useful_tokens_per_second"],
250+
static["useful_tokens_per_second"]),
251+
"avoided_decode_tokens": static["computed_tokens"] - continuous["computed_tokens"],
252+
},
253+
},
254+
}
255+
256+
257+
def run(args):
258+
validate_args(args)
259+
import os
260+
import torch
261+
from transformers import AutoModelForCausalLM, AutoTokenizer
262+
import deepspeed
263+
from deepspeed.runtime.rollout.hybrid_engine_rollout import HybridEngineRollout, HybridEngineRolloutConfig
264+
265+
if not torch.cuda.is_available():
266+
raise RuntimeError("continuous batching benchmark requires CUDA")
267+
if int(os.getenv("WORLD_SIZE", "1")) != 1:
268+
raise RuntimeError("this benchmark supports exactly one GPU process")
269+
torch.manual_seed(args.seed)
270+
device = torch.device("cuda", int(os.getenv("LOCAL_RANK", "0")))
271+
torch.cuda.set_device(device)
272+
dtype = torch.float16 if args.dtype == "fp16" else torch.bfloat16
273+
274+
tokenizer = AutoTokenizer.from_pretrained(args.model)
275+
pad_token_id = tokenizer.pad_token_id
276+
if pad_token_id is None:
277+
if tokenizer.eos_token_id is None:
278+
raise ValueError("tokenizer must define pad_token_id or eos_token_id")
279+
pad_token_id = tokenizer.eos_token_id
280+
tokenizer.pad_token = tokenizer.eos_token
281+
# The Core continuous scheduler otherwise treats model EOS as early completion.
282+
tokenizer.eos_token_id = None
283+
model = AutoModelForCausalLM.from_pretrained(args.model, torch_dtype=dtype, low_cpu_mem_usage=True).to(device)
284+
model.eval()
285+
model.config.pad_token_id = pad_token_id
286+
model.config.eos_token_id = None
287+
capacity = args.max_batch_size
288+
# Keep all three modes on the same unmodified HF forward/generate path.
289+
# Enabling DeepSpeed HybridEngine here replaces ``module.generate`` and
290+
# injects inference kernels whose batch-shape-dependent numerics can make
291+
# exact token comparison impossible. Core's rollout API only requires an
292+
# object exposing ``module`` for continuous generation.
293+
from types import SimpleNamespace
294+
engine = SimpleNamespace(module=model)
295+
rollout = HybridEngineRollout(engine, tokenizer, HybridEngineRolloutConfig())
296+
prompts = _synthetic_prompts(model.config.vocab_size, pad_token_id, args.prompt_length,
297+
len(args.response_lengths), device, args.seed)
298+
requests = [_request(prompt) for prompt in prompts]
299+
300+
hf_generate = model.generate
301+
sequential_latencies, references, sequential_peak = _run_sequential(
302+
hf_generate, engine.module.config, requests, args.response_lengths, args, args.prompt_length)
303+
static_latencies, static_peak = _run_static(
304+
hf_generate, engine.module.config, requests, args.response_lengths, args, args.prompt_length, references)
305+
continuous_latencies, continuous_peak = _run_continuous(
306+
rollout, requests, args.response_lengths, args, args.prompt_length, references)
307+
useful_tokens = sum(args.response_lengths)
308+
mode_results = {
309+
"sequential_eager": _mode_result(sequential_latencies, useful_tokens, useful_tokens, sequential_peak),
310+
"static_batch": _mode_result(static_latencies, useful_tokens,
311+
static_computed_tokens(args.response_lengths, capacity), static_peak),
312+
"continuous_batch": _mode_result(continuous_latencies, useful_tokens, useful_tokens, continuous_peak),
313+
}
314+
environment = {"torch": torch.__version__, "cuda": torch.version.cuda,
315+
"transformers": __import__("transformers").__version__,
316+
"deepspeed": deepspeed.__version__, "gpu": torch.cuda.get_device_name(device)}
317+
result = _build_result(args, environment, mode_results, useful_tokens)
318+
output_path = Path(args.output)
319+
output_path.parent.mkdir(parents=True, exist_ok=True)
320+
output_path.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
321+
print(f"Wrote benchmark results to {output_path}")
322+
return result
323+
324+
325+
if __name__ == "__main__":
326+
run(_build_parser().parse_args())

0 commit comments

Comments
 (0)