Skip to content

Commit e02d9de

Browse files
committed
fix(opsd): report low-precision token agreement
Signed-off-by: nathon-lee <leejianwoo@gmail.com>
1 parent d4e1a94 commit e02d9de

3 files changed

Lines changed: 121 additions & 23 deletions

File tree

‎benchmarks/opsd/README.md‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -66,15 +66,21 @@ max_batch_size=...)`, retiring completed rows and admitting pending requests.
6666
All modes use one loaded model, the same deterministic equal-width token-ID
6767
prompts, response budgets, seed, and capacity. Model loading is outside the
6868
timed region. Every measured iteration synchronizes CUDA before and after the
69-
work, resets peak memory statistics at the start of each mode, and checks every
70-
response token against the sequential eager baseline. Generation is greedy
71-
only (`--temperature 0`), uses `min_new_tokens == max_new_tokens`, and disables
72-
EOS (`eos_token_id=None`) so each request consumes its requested budget.
69+
work and resets peak memory statistics at the start of each mode. Responses
70+
must be deterministic across measured iterations within each mode. Because
71+
low-precision kernels are not necessarily invariant to batch and cache shapes,
72+
cross-mode token agreement with the sequential eager baseline is reported
73+
rather than required by default. Pass `--require-exact-token-match` to make a
74+
cross-mode mismatch fail the run. Generation is greedy only (`--temperature
75+
0`), uses `min_new_tokens == max_new_tokens`, and disables EOS
76+
(`eos_token_id=None`) so each request consumes its requested budget.
7377

7478
The JSON `environment` records torch/CUDA/Transformers/DeepSpeed versions and
7579
GPU. `config` records the CLI workload. Each `results` mode reports
7680
`latency_ms` (mean/p50/p95), useful and computed tokens, useful-token
77-
throughput, and peak allocated memory in MiB. Useful tokens are always the sum
81+
throughput, peak allocated memory in MiB, and
82+
`token_agreement_vs_sequential_eager` (matched/total tokens, agreement rate,
83+
exact-match status, and the first mismatch). Useful tokens are always the sum
7884
of response budgets; computed tokens are equal to useful tokens for sequential
7985
and continuous modes, while static batching uses
8086
`sum(group_size * group_max_response_length)`. `comparisons` reports percentage

‎benchmarks/opsd/benchmark_continuous_batching.py‎

Lines changed: 70 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ def _build_parser():
7676
parser.add_argument("--iterations", type=int, default=3)
7777
parser.add_argument("--temperature", type=float, default=0.0)
7878
parser.add_argument("--seed", type=int, default=1234)
79+
parser.add_argument("--require-exact-token-match", action="store_true",
80+
help="fail when static or continuous responses differ from sequential eager")
7981
parser.add_argument("--output", default="opsd_continuous_batching.json")
8082
return parser
8183

@@ -125,9 +127,39 @@ def _response(output_ids, prompt_length, response_length):
125127
return output_ids[0, prompt_length:prompt_length + response_length].detach().cpu()
126128

127129

128-
def _assert_matches(reference, candidate, mode, request_index):
130+
def _assert_matches(reference, candidate, comparison, request_index):
129131
if len(reference) != len(candidate) or not reference.equal(candidate):
130-
raise AssertionError(f"{mode} request {request_index} response tokens differ from sequential_eager")
132+
raise AssertionError(f"{comparison} request {request_index} response tokens differ")
133+
134+
135+
def token_agreement(references, candidates):
136+
"""Summarize token agreement without assuming low-precision batch invariance."""
137+
if len(references) != len(candidates):
138+
raise AssertionError("response counts differ")
139+
matched_tokens = 0
140+
total_tokens = 0
141+
first_mismatch = None
142+
for request_index, (reference, candidate) in enumerate(zip(references, candidates)):
143+
if len(reference) != len(candidate):
144+
raise AssertionError(f"request {request_index} response lengths differ")
145+
matches = reference == candidate
146+
matched_tokens += int(matches.sum().item())
147+
total_tokens += len(reference)
148+
if first_mismatch is None and not bool(matches.all().item()):
149+
response_position = int((~matches).nonzero().flatten()[0].item())
150+
first_mismatch = {
151+
"request_index": request_index,
152+
"response_position": response_position,
153+
"reference_token": int(reference[response_position].item()),
154+
"candidate_token": int(candidate[response_position].item()),
155+
}
156+
return {
157+
"matched_tokens": matched_tokens,
158+
"total_tokens": total_tokens,
159+
"agreement_rate": matched_tokens / total_tokens,
160+
"exact_match": matched_tokens == total_tokens,
161+
"first_mismatch": first_mismatch,
162+
}
131163

132164

133165
def _run_sequential(generate, model_config, requests, response_lengths, args, prompt_length):
@@ -155,12 +187,13 @@ def _run_sequential(generate, model_config, requests, response_lengths, args, pr
155187
reference_responses = outputs
156188
else:
157189
for index, response in enumerate(outputs):
158-
_assert_matches(reference_responses[index], response, "sequential_eager", index)
190+
_assert_matches(reference_responses[index], response, "sequential_eager repeat", index)
159191
return latencies, reference_responses, _peak_memory_mb()
160192

161193

162194
def _run_static(generate, model_config, requests, response_lengths, args, prompt_length, reference_responses):
163195
latencies = []
196+
measured_responses = []
164197
groups = group_request_indices(response_lengths, args.max_batch_size)
165198
pad_token_id = getattr(model_config, "pad_token_id", None)
166199

@@ -186,13 +219,20 @@ def invoke():
186219
outputs = invoke()
187220
_sync()
188221
latencies.append((time.perf_counter() - start) * 1000.0)
189-
for index, response in enumerate(outputs):
190-
_assert_matches(reference_responses[index], response, "static_batch", index)
191-
return latencies, _peak_memory_mb()
222+
if not measured_responses:
223+
measured_responses = outputs
224+
if args.require_exact_token_match:
225+
for index, response in enumerate(outputs):
226+
_assert_matches(reference_responses[index], response, "static_batch vs sequential_eager", index)
227+
else:
228+
for index, response in enumerate(outputs):
229+
_assert_matches(measured_responses[index], response, "static_batch repeat", index)
230+
return latencies, measured_responses, _peak_memory_mb()
192231

193232

194233
def _run_continuous(rollout, requests, response_lengths, args, prompt_length, reference_responses):
195234
latencies = []
235+
measured_responses = []
196236
configs = [_sampling_config(length, args.temperature) for length in response_lengths]
197237
for _ in range(args.warmup):
198238
rollout.generate_continuous(requests, configs, max_batch_size=args.max_batch_size)
@@ -203,13 +243,21 @@ def _run_continuous(rollout, requests, response_lengths, args, prompt_length, re
203243
outputs = rollout.generate_continuous(requests, configs, max_batch_size=args.max_batch_size)
204244
_sync()
205245
latencies.append((time.perf_counter() - start) * 1000.0)
206-
for index, (output, length) in enumerate(zip(outputs, response_lengths)):
207-
response = output.input_ids[0, prompt_length:prompt_length + length].detach().cpu()
208-
_assert_matches(reference_responses[index], response, "continuous_batch", index)
209-
return latencies, _peak_memory_mb()
246+
responses = [output.input_ids[0, prompt_length:prompt_length + length].detach().cpu()
247+
for output, length in zip(outputs, response_lengths)]
248+
if not measured_responses:
249+
measured_responses = responses
250+
if args.require_exact_token_match:
251+
for index, response in enumerate(responses):
252+
_assert_matches(reference_responses[index], response,
253+
"continuous_batch vs sequential_eager", index)
254+
else:
255+
for index, response in enumerate(responses):
256+
_assert_matches(measured_responses[index], response, "continuous_batch repeat", index)
257+
return latencies, measured_responses, _peak_memory_mb()
210258

211259

212-
def _mode_result(latencies, useful_tokens, computed_tokens, peak_memory):
260+
def _mode_result(latencies, useful_tokens, computed_tokens, peak_memory, agreement):
213261
summary = summarize_latencies(latencies)
214262
mean_latency = summary["mean"]
215263
return {
@@ -218,6 +266,7 @@ def _mode_result(latencies, useful_tokens, computed_tokens, peak_memory):
218266
"computed_tokens": computed_tokens,
219267
"useful_tokens_per_second": useful_tokens / (mean_latency / 1000.0),
220268
"peak_memory_mb": peak_memory,
269+
"token_agreement_vs_sequential_eager": agreement,
221270
}
222271

223272

@@ -234,7 +283,8 @@ def _build_result(args, environment, mode_results, useful_tokens):
234283
"config": {"model": args.model, "dtype": args.dtype, "prompt_length": args.prompt_length,
235284
"response_lengths": args.response_lengths, "max_batch_size": args.max_batch_size,
236285
"warmup": args.warmup, "iterations": args.iterations,
237-
"temperature": args.temperature, "seed": args.seed},
286+
"temperature": args.temperature, "seed": args.seed,
287+
"require_exact_token_match": args.require_exact_token_match},
238288
"results": mode_results,
239289
"comparisons": {
240290
"continuous_vs_sequential": {
@@ -300,16 +350,19 @@ def run(args):
300350
hf_generate = model.generate
301351
sequential_latencies, references, sequential_peak = _run_sequential(
302352
hf_generate, engine.module.config, requests, args.response_lengths, args, args.prompt_length)
303-
static_latencies, static_peak = _run_static(
353+
static_latencies, static_responses, static_peak = _run_static(
304354
hf_generate, engine.module.config, requests, args.response_lengths, args, args.prompt_length, references)
305-
continuous_latencies, continuous_peak = _run_continuous(
355+
continuous_latencies, continuous_responses, continuous_peak = _run_continuous(
306356
rollout, requests, args.response_lengths, args, args.prompt_length, references)
307357
useful_tokens = sum(args.response_lengths)
308358
mode_results = {
309-
"sequential_eager": _mode_result(sequential_latencies, useful_tokens, useful_tokens, sequential_peak),
359+
"sequential_eager": _mode_result(sequential_latencies, useful_tokens, useful_tokens, sequential_peak,
360+
token_agreement(references, references)),
310361
"static_batch": _mode_result(static_latencies, useful_tokens,
311-
static_computed_tokens(args.response_lengths, capacity), static_peak),
312-
"continuous_batch": _mode_result(continuous_latencies, useful_tokens, useful_tokens, continuous_peak),
362+
static_computed_tokens(args.response_lengths, capacity), static_peak,
363+
token_agreement(references, static_responses)),
364+
"continuous_batch": _mode_result(continuous_latencies, useful_tokens, useful_tokens, continuous_peak,
365+
token_agreement(references, continuous_responses)),
313366
}
314367
environment = {"torch": torch.__version__, "cuda": torch.version.cuda,
315368
"transformers": __import__("transformers").__version__,

‎benchmarks/opsd/tests/test_benchmark_continuous_batching.py‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,14 @@
44
import argparse
55

66
import pytest
7+
import torch
78

89
from benchmarks.opsd.benchmark_continuous_batching import (
910
_build_result,
1011
group_request_indices,
1112
static_computed_tokens,
1213
summarize_latencies,
14+
token_agreement,
1315
validate_args,
1416
)
1517

@@ -28,6 +30,42 @@ def test_percentile_summary_matches_opsd_nearest_rank():
2830
assert summarize_latencies([4.0, 1.0, 3.0, 2.0]) == {"mean": 2.5, "p50": 2.0, "p95": 4.0}
2931

3032

33+
def test_token_agreement_reports_first_mismatch():
34+
references = [torch.tensor([1, 2]), torch.tensor([5, 343, 9, 1405, 9932])]
35+
candidates = [torch.tensor([1, 2]), torch.tensor([5, 343, 9, 1287, 1422])]
36+
37+
agreement = token_agreement(references, candidates)
38+
39+
assert agreement == {
40+
"matched_tokens": 5,
41+
"total_tokens": 7,
42+
"agreement_rate": 5 / 7,
43+
"exact_match": False,
44+
"first_mismatch": {
45+
"request_index": 1,
46+
"response_position": 3,
47+
"reference_token": 1405,
48+
"candidate_token": 1287,
49+
},
50+
}
51+
52+
53+
def test_token_agreement_reports_exact_match():
54+
responses = [torch.tensor([1, 2, 3])]
55+
assert token_agreement(responses, responses) == {
56+
"matched_tokens": 3,
57+
"total_tokens": 3,
58+
"agreement_rate": 1.0,
59+
"exact_match": True,
60+
"first_mismatch": None,
61+
}
62+
63+
64+
def test_token_agreement_rejects_length_mismatch():
65+
with pytest.raises(AssertionError, match="response lengths differ"):
66+
token_agreement([torch.tensor([1, 2])], [torch.tensor([1])])
67+
68+
3169
def test_validate_args_rejects_non_greedy():
3270
args = argparse.Namespace(dtype="fp16", prompt_length=4, response_lengths=[2], max_batch_size=1,
3371
warmup=0, iterations=1, temperature=0.2, seed=1)
@@ -43,7 +81,8 @@ def test_validate_args_accepts_defaults_shape():
4381

4482
def test_result_comparisons_use_mean_latency_and_useful_throughput():
4583
args = argparse.Namespace(model="m", dtype="fp16", prompt_length=8, response_lengths=[2, 4],
46-
max_batch_size=1, warmup=1, iterations=1, temperature=0.0, seed=1)
84+
max_batch_size=1, warmup=1, iterations=1, temperature=0.0, seed=1,
85+
require_exact_token_match=False)
4786
modes = {
4887
"sequential_eager": {"latency_ms": {"mean": 10.0, "p50": 10.0, "p95": 10.0},
4988
"useful_tokens": 6, "computed_tokens": 6,

0 commit comments

Comments
 (0)