feat: encode_special_tokens flag

bindings/node/index.d.ts

@@ -27,6 +27,11 @@ export interface EncodeOptions {
    * when left out.
    */
   addSpecialTokens?: boolean
+  /**
+   * Whether a special token written in the text goes through the model (`true`) or becomes its
+   * added-vocabulary id. `false` when left out.
+   */
+  encodeSpecialTokens?: boolean
   /**
    * Padding for this call, replacing the tokenizer's configured padding. `false` disables
    * padding. Left out, the configured padding applies.

bindings/node/src/pipeline.rs

@@ -27,6 +27,9 @@ pub struct EncodeOptions {
   /// Whether the post-processor adds its special tokens, such as `[CLS]` and `[SEP]`. `true`
   /// when left out.
   pub add_special_tokens: Option<bool>,
+  /// Whether a special token written in the text goes through the model (`true`) or becomes its
+  /// added-vocabulary id. `false` when left out.
+  pub encode_special_tokens: Option<bool>,
   /// Padding for this call, replacing the tokenizer's configured padding. `false` disables
   /// padding. Left out, the configured padding applies.
   #[napi(ts_type = "false | PaddingOptions")]
@@ -159,6 +162,7 @@ impl PipelineTokenizer {
     let options = options.unwrap_or_default();
     Ok(PipelineEncodeOptions {
       add_special_tokens: options.add_special_tokens.unwrap_or(true),
+      encode_special_tokens: options.encode_special_tokens.unwrap_or(false),
       padding: override_with(options.padding, PaddingOptions::params)?,
       truncation: override_with(options.truncation, TruncationOptions::params)?,
     })

bindings/node/test/pipeline.test.ts

@@ -69,6 +69,18 @@ test('addSpecialTokens is honoured', () => {
   assert.ok(withSpecials.length >= without.length)
 })
 
+test('encodeSpecialTokens sends a special token through the model', () => {
+  const tok = PipelineTokenizer.fromFile(MODEL)
+  const sepId = JSON.parse(readFileSync(MODEL, 'utf8')).added_tokens.find(
+    (t: { content: string; id: number }) => t.content === '[SEP]',
+  ).id
+  const carved = tok.encode('[SEP]', { addSpecialTokens: false })
+  const encoded = tok.encode('[SEP]', { addSpecialTokens: false, encodeSpecialTokens: true })
+  assert.deepStrictEqual(carved, Uint32Array.of(sepId))
+  // `Whitespace` cuts `[SEP]` into `[`, `SEP`, `]`: the same pieces the spaced spelling gives.
+  assert.deepStrictEqual(encoded, tok.encode('[ SEP ]', { addSpecialTokens: false }))
+})
+
 test('padding.length pads to a fixed length', () => {
   const tok = PipelineTokenizer.fromFile(MODEL)
   const plain = tok.encode('Hello')

bindings/python/python/tokenizers/tokenizers.pyi

@@ -189,6 +189,7 @@ class Tokenizer:
         text: str,
         *,
         add_special_tokens: bool = True,
+        encode_special_tokens: bool = False,
         padding: Padding | None = ...,
         truncation: Truncation | None = ...,
     ) -> Encoding:
@@ -200,6 +201,9 @@ class Tokenizer:
                 The text to encode.
             add_special_tokens: bool
                  Whether the post-processor adds its special tokens, such as `[CLS]` and `[SEP]`.
+            encode_special_tokens: bool
+                Whether special tokens should be encoded, ie go through the tokenizer model (`True`)
+                or be replaced by their id in the added vocabulary.
             padding: `Padding` or `None`
                 Padding options. Pass `None` to disable padding.
                 When omitted, defaults to the padding options configured on the tokenizer.
@@ -216,6 +220,7 @@ class Tokenizer:
         texts: Sequence[str],
         *,
         add_special_tokens: bool = True,
+        encode_special_tokens: bool = True,
         padding: Padding | None = ...,
         truncation: Truncation | None = ...,
     ) -> list[Encoding]:
@@ -228,6 +233,9 @@ class Tokenizer:
                 The batch of text to encode.
             add_special_tokens: bool
                 Whether the post-processor adds its special tokens, such as `[CLS]` and `[SEP]`.
+            encode_special_tokens: bool
+                Whether special tokens should be encoded, ie go through the tokenizer model (`True`)
+                or be replaced by their id in the added vocabulary.
             padding: `Padding` or `None`
                 Padding options. Pass `None` to disable padding.
                 When omitted, defaults to the padding options configured on the tokenizer.
@@ -319,6 +327,7 @@ class Tokenizer:
         text: str,
         *,
         add_special_tokens: bool = True,
+        encode_special_tokens: bool = False,
         padding: Padding | None = ...,
         truncation: Truncation | None = ...,
     ) -> list[str]:
@@ -333,6 +342,9 @@ class Tokenizer:
                 The text to tokenize.
             add_special_tokens: bool
                  Whether the post-processor adds its special tokens, such as `[CLS]` and `[SEP]`.
+            encode_special_tokens: bool
+                Whether special tokens should be encoded, ie go through the tokenizer model (`True`)
+                or be replaced by their id in the added vocabulary.
             padding: `Padding` or `None`
                 Padding options. Pass `None` to disable padding.
                 When omitted, defaults to the padding options configured on the tokenizer.

bindings/python/src/tokenizer.rs

@@ -27,6 +27,7 @@ impl Tokenizer {
     fn make_options(
         &self,
         add_special_tokens: bool,
+        encode_special_tokens: bool,
         padding: OverrideSentinel<PaddingParams>,
         truncation: OverrideSentinel<TruncationParams>,
     ) -> PyResult<EncodeOptions> {
@@ -46,6 +47,7 @@ impl Tokenizer {
         };
         Ok(EncodeOptions {
             add_special_tokens,
+            encode_special_tokens,
             padding,
             truncation,
         })
@@ -194,6 +196,9 @@ impl Tokenizer {
     ///         The text to encode.
     ///     add_special_tokens: bool
     ///          Whether the post-processor adds its special tokens, such as `[CLS]` and `[SEP]`.
+    ///     encode_special_tokens: bool
+    ///         Whether special tokens should be encoded, ie go through the tokenizer model (`True`)
+    ///         or be replaced by their id in the added vocabulary.
     ///     padding: `Padding` or `None`
     ///         Padding options. Pass `None` to disable padding.
     ///         When omitted, defaults to the padding options configured on the tokenizer.
@@ -203,16 +208,22 @@ impl Tokenizer {
     ///
     /// Returns:
     ///     Encoding
-    #[pyo3(signature = (text, *, add_special_tokens=true, padding=OverrideSentinel::<PaddingParams>::InheritConfig, truncation=OverrideSentinel::<TruncationParams>::InheritConfig))]
+    #[pyo3(signature = (text, *, add_special_tokens=true, encode_special_tokens=false, padding=OverrideSentinel::<PaddingParams>::InheritConfig, truncation=OverrideSentinel::<TruncationParams>::InheritConfig))]
     fn encode(
         &self,
         py: Python<'_>,
         text: String,
         add_special_tokens: bool,
+        encode_special_tokens: bool,
         padding: OverrideSentinel<PaddingParams>,
         truncation: OverrideSentinel<TruncationParams>,
     ) -> PyResult<Encoding> {
-        let options = self.make_options(add_special_tokens, padding, truncation)?;
+        let options = self.make_options(
+            add_special_tokens,
+            encode_special_tokens,
+            padding,
+            truncation,
+        )?;
         // py.detach releases the GIL while encode runs on Rust side
         let encodings = py
             .detach(|| self.pipeline.encode(text, &options).wait())
@@ -230,6 +241,9 @@ impl Tokenizer {
     ///         The text to tokenize.
     ///     add_special_tokens: bool
     ///          Whether the post-processor adds its special tokens, such as `[CLS]` and `[SEP]`.
+    ///     encode_special_tokens: bool
+    ///         Whether special tokens should be encoded, ie go through the tokenizer model (`True`)
+    ///         or be replaced by their id in the added vocabulary.
     ///     padding: `Padding` or `None`
     ///         Padding options. Pass `None` to disable padding.
     ///         When omitted, defaults to the padding options configured on the tokenizer.
@@ -239,16 +253,22 @@ impl Tokenizer {
     ///
     /// Returns:
     ///     list[str]
-    #[pyo3(signature = (text, *, add_special_tokens=true, padding=OverrideSentinel::<PaddingParams>::InheritConfig, truncation=OverrideSentinel::<TruncationParams>::InheritConfig))]
+    #[pyo3(signature = (text, *, add_special_tokens=true, encode_special_tokens=false, padding=OverrideSentinel::<PaddingParams>::InheritConfig, truncation=OverrideSentinel::<TruncationParams>::InheritConfig))]
     fn tokenize(
         &self,
         py: Python<'_>,
         text: String,
         add_special_tokens: bool,
+        encode_special_tokens: bool,
         padding: OverrideSentinel<PaddingParams>,
         truncation: OverrideSentinel<TruncationParams>,
     ) -> PyResult<Vec<String>> {
-        let options = self.make_options(add_special_tokens, padding, truncation)?;
+        let options = self.make_options(
+            add_special_tokens,
+            encode_special_tokens,
+            padding,
+            truncation,
+        )?;
         py.detach(|| -> tk_encode::Result<Vec<String>> {
             let encodings = self.pipeline.encode(text, &options).wait()?;
             let ids: Vec<u32> = encodings[0].ids().iter().map(|token| token.id()).collect();
@@ -265,6 +285,9 @@ impl Tokenizer {
     ///         The batch of text to encode.
     ///     add_special_tokens: bool
     ///         Whether the post-processor adds its special tokens, such as `[CLS]` and `[SEP]`.
+    ///     encode_special_tokens: bool
+    ///         Whether special tokens should be encoded, ie go through the tokenizer model (`True`)
+    ///         or be replaced by their id in the added vocabulary.
     ///     padding: `Padding` or `None`
     ///         Padding options. Pass `None` to disable padding.
     ///         When omitted, defaults to the padding options configured on the tokenizer.
@@ -274,16 +297,22 @@ impl Tokenizer {
     ///
     /// Returns:
     ///     List[Encoding]
-    #[pyo3(signature = (texts, *, add_special_tokens=true, padding=OverrideSentinel::<PaddingParams>::InheritConfig, truncation=OverrideSentinel::<TruncationParams>::InheritConfig))]
+    #[pyo3(signature = (texts, *, add_special_tokens=true, encode_special_tokens=true, padding=OverrideSentinel::<PaddingParams>::InheritConfig, truncation=OverrideSentinel::<TruncationParams>::InheritConfig))]
     fn encode_batch(
         &self,
         py: Python<'_>,
         texts: Vec<String>,
         add_special_tokens: bool,
+        encode_special_tokens: bool,
         padding: OverrideSentinel<PaddingParams>,
         truncation: OverrideSentinel<TruncationParams>,
     ) -> PyResult<Vec<Encoding>> {
-        let options = self.make_options(add_special_tokens, padding, truncation)?;
+        let options = self.make_options(
+            add_special_tokens,
+            encode_special_tokens,
+            padding,
+            truncation,
+        )?;
         // py.detach releases the GIL while encode runs on Rust side
         let encodings = py
             .detach(|| self.pipeline.encode(texts, &options).wait())

tokenizers/tk-convert/tests/oracle.rs

@@ -124,6 +124,13 @@ fn cells() -> Vec<(&'static str, EncodeOptions)> {
     vec![
         ("no specials", EncodeOptions::no_specials()),
         ("specials", EncodeOptions::default()),
+        (
+            "encode specials",
+            EncodeOptions {
+                encode_special_tokens: true,
+                ..EncodeOptions::no_specials()
+            },
+        ),
         (
             "truncate right",
             EncodeOptions {
@@ -172,10 +179,12 @@ fn cells() -> Vec<(&'static str, EncodeOptions)> {
     ]
 }
 
-/// `base` with the padding and truncation `options` resolves to: the file's own settings when
-/// inherited, none when off, and `options`' own when given.
+/// `base` with `options` applied: `encode_special_tokens` as given, and the padding and truncation
+/// each resolved to the file's own settings when inherited, none when off, and `options`' own
+/// when given.
 fn released_with(base: &Released, options: &EncodeOptions) -> Released {
     let mut released = base.clone();
+    released.set_encode_special_tokens(options.encode_special_tokens);
     match &options.padding {
         Override::InheritConfig => {}
         Override::Off => {

tokenizers/tk-encode/src/tokenizer/pipeline/encode_options.rs

@@ -2,7 +2,13 @@ use crate::{PaddingParams, TruncationParams};
 
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct EncodeOptions {
+    /// Whether the post-processor should add special tokens to the sequence,
+    /// e.g. `[CLS]`, `[SEP]`, `<|endoftext|>`, `</s>`. Defaults to `true`.
     pub add_special_tokens: bool,
+    /// Whether special tokens found in the sequence should go through the tokenizer model (`true`)
+    /// or be replaced by their id in the vocabulary (`false`). Defaults to `true`.
+    pub encode_special_tokens: bool,
+    /// Override the tokenizer's padding options. Defaults to [`Override::InheritConfig`].
     pub padding: Override<PaddingParams>,
     pub truncation: Override<TruncationParams>,
 }
@@ -11,6 +17,7 @@ impl Default for EncodeOptions {
     fn default() -> Self {
         Self {
             add_special_tokens: true,
+            encode_special_tokens: true,
             padding: Override::InheritConfig,
             truncation: Override::InheritConfig,
         }