@@ -76,6 +76,8 @@ def _build_parser():
7676 parser .add_argument ("--iterations" , type = int , default = 3 )
7777 parser .add_argument ("--temperature" , type = float , default = 0.0 )
7878 parser .add_argument ("--seed" , type = int , default = 1234 )
79+ parser .add_argument ("--require-exact-token-match" , action = "store_true" ,
80+ help = "fail when static or continuous responses differ from sequential eager" )
7981 parser .add_argument ("--output" , default = "opsd_continuous_batching.json" )
8082 return parser
8183
@@ -125,9 +127,39 @@ def _response(output_ids, prompt_length, response_length):
125127 return output_ids [0 , prompt_length :prompt_length + response_length ].detach ().cpu ()
126128
127129
128- def _assert_matches (reference , candidate , mode , request_index ):
130+ def _assert_matches (reference , candidate , comparison , request_index ):
129131 if len (reference ) != len (candidate ) or not reference .equal (candidate ):
130- raise AssertionError (f"{ mode } request { request_index } response tokens differ from sequential_eager" )
132+ raise AssertionError (f"{ comparison } request { request_index } response tokens differ" )
133+
134+
135+ def token_agreement (references , candidates ):
136+ """Summarize token agreement without assuming low-precision batch invariance."""
137+ if len (references ) != len (candidates ):
138+ raise AssertionError ("response counts differ" )
139+ matched_tokens = 0
140+ total_tokens = 0
141+ first_mismatch = None
142+ for request_index , (reference , candidate ) in enumerate (zip (references , candidates )):
143+ if len (reference ) != len (candidate ):
144+ raise AssertionError (f"request { request_index } response lengths differ" )
145+ matches = reference == candidate
146+ matched_tokens += int (matches .sum ().item ())
147+ total_tokens += len (reference )
148+ if first_mismatch is None and not bool (matches .all ().item ()):
149+ response_position = int ((~ matches ).nonzero ().flatten ()[0 ].item ())
150+ first_mismatch = {
151+ "request_index" : request_index ,
152+ "response_position" : response_position ,
153+ "reference_token" : int (reference [response_position ].item ()),
154+ "candidate_token" : int (candidate [response_position ].item ()),
155+ }
156+ return {
157+ "matched_tokens" : matched_tokens ,
158+ "total_tokens" : total_tokens ,
159+ "agreement_rate" : matched_tokens / total_tokens ,
160+ "exact_match" : matched_tokens == total_tokens ,
161+ "first_mismatch" : first_mismatch ,
162+ }
131163
132164
133165def _run_sequential (generate , model_config , requests , response_lengths , args , prompt_length ):
@@ -155,12 +187,13 @@ def _run_sequential(generate, model_config, requests, response_lengths, args, pr
155187 reference_responses = outputs
156188 else :
157189 for index , response in enumerate (outputs ):
158- _assert_matches (reference_responses [index ], response , "sequential_eager" , index )
190+ _assert_matches (reference_responses [index ], response , "sequential_eager repeat " , index )
159191 return latencies , reference_responses , _peak_memory_mb ()
160192
161193
162194def _run_static (generate , model_config , requests , response_lengths , args , prompt_length , reference_responses ):
163195 latencies = []
196+ measured_responses = []
164197 groups = group_request_indices (response_lengths , args .max_batch_size )
165198 pad_token_id = getattr (model_config , "pad_token_id" , None )
166199
@@ -186,13 +219,20 @@ def invoke():
186219 outputs = invoke ()
187220 _sync ()
188221 latencies .append ((time .perf_counter () - start ) * 1000.0 )
189- for index , response in enumerate (outputs ):
190- _assert_matches (reference_responses [index ], response , "static_batch" , index )
191- return latencies , _peak_memory_mb ()
222+ if not measured_responses :
223+ measured_responses = outputs
224+ if args .require_exact_token_match :
225+ for index , response in enumerate (outputs ):
226+ _assert_matches (reference_responses [index ], response , "static_batch vs sequential_eager" , index )
227+ else :
228+ for index , response in enumerate (outputs ):
229+ _assert_matches (measured_responses [index ], response , "static_batch repeat" , index )
230+ return latencies , measured_responses , _peak_memory_mb ()
192231
193232
194233def _run_continuous (rollout , requests , response_lengths , args , prompt_length , reference_responses ):
195234 latencies = []
235+ measured_responses = []
196236 configs = [_sampling_config (length , args .temperature ) for length in response_lengths ]
197237 for _ in range (args .warmup ):
198238 rollout .generate_continuous (requests , configs , max_batch_size = args .max_batch_size )
@@ -203,13 +243,21 @@ def _run_continuous(rollout, requests, response_lengths, args, prompt_length, re
203243 outputs = rollout .generate_continuous (requests , configs , max_batch_size = args .max_batch_size )
204244 _sync ()
205245 latencies .append ((time .perf_counter () - start ) * 1000.0 )
206- for index , (output , length ) in enumerate (zip (outputs , response_lengths )):
207- response = output .input_ids [0 , prompt_length :prompt_length + length ].detach ().cpu ()
208- _assert_matches (reference_responses [index ], response , "continuous_batch" , index )
209- return latencies , _peak_memory_mb ()
246+ responses = [output .input_ids [0 , prompt_length :prompt_length + length ].detach ().cpu ()
247+ for output , length in zip (outputs , response_lengths )]
248+ if not measured_responses :
249+ measured_responses = responses
250+ if args .require_exact_token_match :
251+ for index , response in enumerate (responses ):
252+ _assert_matches (reference_responses [index ], response ,
253+ "continuous_batch vs sequential_eager" , index )
254+ else :
255+ for index , response in enumerate (responses ):
256+ _assert_matches (measured_responses [index ], response , "continuous_batch repeat" , index )
257+ return latencies , measured_responses , _peak_memory_mb ()
210258
211259
212- def _mode_result (latencies , useful_tokens , computed_tokens , peak_memory ):
260+ def _mode_result (latencies , useful_tokens , computed_tokens , peak_memory , agreement ):
213261 summary = summarize_latencies (latencies )
214262 mean_latency = summary ["mean" ]
215263 return {
@@ -218,6 +266,7 @@ def _mode_result(latencies, useful_tokens, computed_tokens, peak_memory):
218266 "computed_tokens" : computed_tokens ,
219267 "useful_tokens_per_second" : useful_tokens / (mean_latency / 1000.0 ),
220268 "peak_memory_mb" : peak_memory ,
269+ "token_agreement_vs_sequential_eager" : agreement ,
221270 }
222271
223272
@@ -234,7 +283,8 @@ def _build_result(args, environment, mode_results, useful_tokens):
234283 "config" : {"model" : args .model , "dtype" : args .dtype , "prompt_length" : args .prompt_length ,
235284 "response_lengths" : args .response_lengths , "max_batch_size" : args .max_batch_size ,
236285 "warmup" : args .warmup , "iterations" : args .iterations ,
237- "temperature" : args .temperature , "seed" : args .seed },
286+ "temperature" : args .temperature , "seed" : args .seed ,
287+ "require_exact_token_match" : args .require_exact_token_match },
238288 "results" : mode_results ,
239289 "comparisons" : {
240290 "continuous_vs_sequential" : {
@@ -300,16 +350,19 @@ def run(args):
300350 hf_generate = model .generate
301351 sequential_latencies , references , sequential_peak = _run_sequential (
302352 hf_generate , engine .module .config , requests , args .response_lengths , args , args .prompt_length )
303- static_latencies , static_peak = _run_static (
353+ static_latencies , static_responses , static_peak = _run_static (
304354 hf_generate , engine .module .config , requests , args .response_lengths , args , args .prompt_length , references )
305- continuous_latencies , continuous_peak = _run_continuous (
355+ continuous_latencies , continuous_responses , continuous_peak = _run_continuous (
306356 rollout , requests , args .response_lengths , args , args .prompt_length , references )
307357 useful_tokens = sum (args .response_lengths )
308358 mode_results = {
309- "sequential_eager" : _mode_result (sequential_latencies , useful_tokens , useful_tokens , sequential_peak ),
359+ "sequential_eager" : _mode_result (sequential_latencies , useful_tokens , useful_tokens , sequential_peak ,
360+ token_agreement (references , references )),
310361 "static_batch" : _mode_result (static_latencies , useful_tokens ,
311- static_computed_tokens (args .response_lengths , capacity ), static_peak ),
312- "continuous_batch" : _mode_result (continuous_latencies , useful_tokens , useful_tokens , continuous_peak ),
362+ static_computed_tokens (args .response_lengths , capacity ), static_peak ,
363+ token_agreement (references , static_responses )),
364+ "continuous_batch" : _mode_result (continuous_latencies , useful_tokens , useful_tokens , continuous_peak ,
365+ token_agreement (references , continuous_responses )),
313366 }
314367 environment = {"torch" : torch .__version__ , "cuda" : torch .version .cuda ,
315368 "transformers" : __import__ ("transformers" ).__version__ ,
0 commit comments