From 2c42cda5d38384962c8da79051052d20defe73a3 Mon Sep 17 00:00:00 2001
From: Bog <bogsacha@gmail.com>
Date: Tue, 1 Sep 2026 23:19:01 +0200
Subject: [PATCH] perf(pre_tokenizers): optimize ByteLevel decoder memory
 allocations & indexing

Reduce memory allocations in ByteLevel::decode_chain by pre-allocating a single
output buffer for the total sequence length instead of allocating a temporary Vec<u8>
for every single token (try_fold(vec![], ...)).

Key Improvements:
- Allocation Reduction: Replaces N dynamic heap allocations per decode call with 1 pre-allocated buffer.
- O(1) Array Indexing: Replaces HashMap byte lookup with direct [char; 256] array indexing in BYTES_CHAR.
- Derived Mapping: CHAR_BYTES is derived dynamically from BYTES_CHAR in a single line, eliminating duplicate map construction without hardcoded range constants.

Benchmark Results (Criterion decode-llama3-(en|ja)/decode):
- Llama 3 English decode: ~35% to 40% faster (~95-102 ms vs 157.8 ms baseline)
- Llama 3 Japanese decode: ~21% to 25% faster (~18.8 ms vs 25.2 ms baseline)

AI-assisted change.
---
 tokenizers/src/pre_tokenizers/byte_level.rs | 71 +++++++++++++++------
 1 file changed, 51 insertions(+), 20 deletions(-)

diff --git a/tokenizers/src/pre_tokenizers/byte_level.rs b/tokenizers/src/pre_tokenizers/byte_level.rs
index 8bc0f30af0..94f1a77091 100644
--- a/tokenizers/src/pre_tokenizers/byte_level.rs
+++ b/tokenizers/src/pre_tokenizers/byte_level.rs
@@ -44,9 +44,13 @@ static RE: LazyLock<SysRegex> = LazyLock::new(|| {
     SysRegex::new(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+")
         .unwrap()
 });
-static BYTES_CHAR: LazyLock<AHashMap<u8, char>> = LazyLock::new(bytes_char);
+static BYTES_CHAR: LazyLock<[char; 256]> = LazyLock::new(|| {
+    let map = bytes_char();
+    std::array::from_fn(|i| map[&(i as u8)])
+});
+
 static CHAR_BYTES: LazyLock<AHashMap<char, u8>> =
-    LazyLock::new(|| bytes_char().into_iter().map(|(c, b)| (b, c)).collect());
+    LazyLock::new(|| BYTES_CHAR.iter().enumerate().map(|(b, &c)| (c, b as u8)).collect());
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 /// Provides all the necessary steps to handle the BPE tokenization at the byte-level. Takes care
@@ -91,7 +95,7 @@ impl ByteLevel {
     }
 
     pub fn alphabet() -> AHashSet<char> {
-        BYTES_CHAR.values().copied().collect()
+        BYTES_CHAR.iter().copied().collect()
     }
 
     #[must_use]
@@ -138,7 +142,7 @@ impl PreTokenizer for ByteLevel {
                     s.as_bytes()[i..i + size]
                         .iter()
                         .enumerate()
-                        .map(|(i, b)| (BYTES_CHAR[b], isize::from(i > 0))),
+                        .map(|(i, b)| (BYTES_CHAR[*b as usize], isize::from(i > 0))),
                 );
             }
             normalized.transform(transformations, 0);
@@ -154,20 +158,23 @@ impl PreTokenizer for ByteLevel {
 /// as String.
 impl Decoder for ByteLevel {
     fn decode_chain(&self, tokens: Vec<String>) -> Result<Vec<String>> {
-        let toks = tokens
-            .into_iter()
-            .flat_map(|t| {
-                t.chars()
-                    .try_fold(vec![], |mut acc, c| {
-                        CHAR_BYTES.get(&c).map(|b| {
-                            acc.push(*b);
-                            acc
-                        })
-                    })
-                    .unwrap_or_else(|| t.as_bytes().to_vec())
-            })
-            .collect::<Vec<u8>>();
-        Ok(vec![String::from_utf8_lossy(&toks).to_string()])
+        let mut bytes = Vec::with_capacity(tokens.iter().map(String::len).sum());
+        for token in &tokens {
+            let start = bytes.len();
+            let decoded = token.chars().all(|c| match CHAR_BYTES.get(&c) {
+                Some(&b) => {
+                    bytes.push(b);
+                    true
+                }
+                None => false,
+            });
+
+            if !decoded {
+                bytes.truncate(start);
+                bytes.extend_from_slice(token.as_bytes());
+            }
+        }
+        Ok(vec![String::from_utf8_lossy(&bytes).into_owned()])
     }
 }
 
@@ -203,12 +210,12 @@ pub fn process_offsets(encoding: &mut Encoding, add_prefix_space: bool) {
     encoding.process_tokens_with_offsets_mut(|(i, (token, offsets))| {
         let mut leading_spaces = token
             .chars()
-            .take_while(|c| *c == BYTES_CHAR[&b' '] || c.is_whitespace())
+            .take_while(|c| *c == BYTES_CHAR[b' ' as usize] || c.is_whitespace())
             .count();
         let trailing_spaces = token
             .chars()
             .rev()
-            .take_while(|c| *c == BYTES_CHAR[&b' '] || c.is_whitespace())
+            .take_while(|c| *c == BYTES_CHAR[b' ' as usize] || c.is_whitespace())
             .count();
 
         if leading_spaces > 0 || trailing_spaces > 0 {
@@ -568,6 +575,30 @@ mod tests {
         );
     }
 
+    #[test]
+    fn decode_empty_tokens() {
+        let byte_level = ByteLevel::default();
+        assert_eq!(byte_level.decode_chain(vec![]).unwrap(), vec![""]);
+    }
+
+    #[test]
+    fn decode_partial_unknown_token() {
+        let byte_level = ByteLevel::default();
+        assert_eq!(
+            byte_level
+                .decode_chain(vec!["Hello".into(), "Ġworld☺".into()])
+                .unwrap(),
+            vec!["HelloĠworld☺"]
+        );
+    }
+
+    #[test]
+    fn char_bytes_round_trip() {
+        for byte in 0..=u8::MAX {
+            assert_eq!(CHAR_BYTES.get(&BYTES_CHAR[byte as usize]), Some(&byte));
+        }
+    }
+
     #[test]
     fn deserialization() {
         // Before use_regex