Support ByteLevel encoding in Bpe tokenizer to support DeepSeek model

eng/Versions.props

@@ -100,7 +100,7 @@
  <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
  <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
  <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
- <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25126.1</MicrosoftMLTestTokenizersVersion>
+ <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25161.1</MicrosoftMLTestTokenizersVersion>
  <SystemDataSqlClientVersion>4.9.0</SystemDataSqlClientVersion>
  <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
  <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>

src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs

@@ -0,0 +1,100 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+
+namespace Microsoft.ML.Tokenizers
+{
+ /// <summary>
+ /// Options for the BPE tokenizer.
+ /// </summary>
+ public sealed class BpeOptions
+ {
+ /// <summary>
+ /// Initializes a new instance of the <see cref="BpeOptions"/> class.
+ /// </summary>
+ public BpeOptions(IEnumerable<(string Token, int Id)> vocabulary)
+ {
+ if (vocabulary == null)
+ {
+ throw new ArgumentNullException(nameof(vocabulary));
+ }
+
+ Vocabulary = vocabulary;
+ }
+
+ /// <summary>
+ /// Gets or sets the vocabulary to use.
+ /// </summary>
+ public IEnumerable<(string Token, int Id)> Vocabulary { get; }
+
+ /// <summary>
+ /// Gets or sets the list of the merge strings used to merge tokens during encoding.
+ /// </summary>
+ public IEnumerable<string>? Merges { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional special tokens to use.
+ /// </summary>
+ public Dictionary<string, int>? SpecialTokens { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional normalizer to normalize the input text before encoding it.
+ /// </summary>
+ public Normalizer? Normalizer { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional pre-tokenizer to split the input text into tokens before encoding it.
+ /// </summary>
+ public PreTokenizer? PreTokenizer { get; set; }
+
+ /// <summary>
+ /// Gets or sets the Unknown token.
+ /// </summary>
+ public string? UnknownToken { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value indicating whether to merge the sequence of the unknown tokens together.
+ /// </summary>
+ public bool FuseUnknownTokens { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional prefix to be used for every subword that is not a beginning-of-word token
+ /// </summary>
+ public string? ContinuingSubwordPrefix { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional suffix to characterize the end-of-word and sub-word
+ /// </summary>
+ public string? EndOfWordSuffix { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value indicating whether to handle the input text in byte level.
+ /// if true, the input text will be converted to UTF-8 bytes before encoding it.
+ /// Additionally, some ASCII characters will be transformed to different characters (e.g Space character will be transformed to 'Ġ' character).
+ /// </summary>
+ public bool ByteLevel { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional beginning of sentence token to be used when encoding the input text.
+ /// </summary>
+ /// <remarks>
+ /// When specified, this token will be added to the beginning of the input text before encoding it.
+ /// This is useful for models that require a specific token to indicate the start of a sentence.
+ /// This token should be present in the vocabulary.
+ /// </remarks>
+ public string? BeginningOfSentenceToken { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional end of sentence token to be used when encoding the input text.
+ /// </summary>
+ /// <remarks>
+ /// When specified, this token will be added to the end of the input text before encoding it.
+ /// This is useful for models that require a specific token to indicate the end of a sentence.
+ /// This token should be present in the vocabulary.
+ /// </remarks>
+ public string? EndOfSentenceToken { get; set; }
+ }
+}

src/Microsoft.ML.Tokenizers/Model/CodeGenTokenizer.cs

@@ -1263,7 +1263,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(BeginningOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(BeginningOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1272,7 +1272,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(EndOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(EndOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1281,7 +1281,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(UnknownToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(UnknownToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1306,7 +1306,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  ReadOnlySpan<char> span = firstToken && hasPrefixSpace && s.Length > 0 && s[0] == _transformedSpace ? s.AsSpan(1) : s.AsSpan();
  firstToken = false;
- AppendToBytesArray(span, ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(span, ref bytes, ref bytesIndex);
  }
  }
 
@@ -1564,27 +1564,6 @@ public OperationStatus Decode(IEnumerable<int> ids, Span<char> destination, bool
  return null;
  }
 
- private void AppendToBytesArray(ReadOnlySpan<char> text, ref byte[] bytes, ref int bytesIndex)
- {
- IReadOnlyDictionary<char, char> unicodeToByte = ByteToUnicodeEncoding.Instance.UnicodeToByte;
- for (int i = 0; i < text.Length; i++)
- {
- if (unicodeToByte.TryGetValue(text[i], out char c))
- {
- if (bytesIndex >= bytes.Length)
- {
- Helpers.ArrayPoolGrow<byte>(ref bytes, bytes.Length * 2);
- }
-
- bytes[bytesIndex++] = (byte)c;
- continue;
- }
-
- // rare cases
- i += Helpers.EncodeCodePointToUtf8(text, i, ref bytes, ref bytesIndex) - 1;
- }
- }
-
  //
  // Private & Internal methods
  //

src/Microsoft.ML.Tokenizers/Model/Word.cs

@@ -289,15 +289,31 @@ public override string ToString()
  return sb.ToString();
  }
 
- public void ToTokens(SortedDictionary<int, string> vocabReverse, List<EncodedToken> tokens, int offset)
+ public void ToTokens(SortedDictionary<int, string> vocabReverse, List<EncodedToken> tokens, int offset, ReadOnlySpan<int> mapping)
  {
  int index = 0;
 
- for (int i = 0; i < SymbolsCount; i++)
+ if (mapping.IsEmpty)
+ {
+ for (int i = 0; i < SymbolsCount; i++)
+ {
+ int endIndex = index + _symbols[i].Len;
+ tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(index + offset, index + offset + _symbols[i].Len)));
+ index += _symbols[i].Len;
+ }
+ }
+ else
  {
- int endIndex = index + _symbols[i].Len;
- tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(index + offset, index + offset + _symbols[i].Len)));
- index += _symbols[i].Len;
+ for (int i = 0; i < SymbolsCount; i++)
+ {
+ int endIndex = index + _symbols[i].Len;
+
+ int mappedIndex = mapping[index];
+ int mappedEndIndex = mapping[endIndex - 1] + 1;
+
+ tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(mappedIndex + offset, mappedEndIndex + offset)));
+ index += _symbols[i].Len;
+ }
  }
  }
  }

eng/Versions.props

@@ -100,7 +100,7 @@
  <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
  <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
  <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
- <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25126.1</MicrosoftMLTestTokenizersVersion>
+ <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25161.1</MicrosoftMLTestTokenizersVersion>
  <SystemDataSqlClientVersion>4.9.0</SystemDataSqlClientVersion>
  <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
  <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>

src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs

@@ -0,0 +1,100 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+
+namespace Microsoft.ML.Tokenizers
+{
+ /// <summary>
+ /// Options for the BPE tokenizer.
+ /// </summary>
+ public sealed class BpeOptions
+ {
+ /// <summary>
+ /// Initializes a new instance of the <see cref="BpeOptions"/> class.
+ /// </summary>
+ public BpeOptions(IEnumerable<(string Token, int Id)> vocabulary)
+ {
+ if (vocabulary == null)
+ {
+ throw new ArgumentNullException(nameof(vocabulary));
+ }
+
+ Vocabulary = vocabulary;
+ }
+
+ /// <summary>
+ /// Gets or sets the vocabulary to use.
+ /// </summary>
+ public IEnumerable<(string Token, int Id)> Vocabulary { get; }
+
+ /// <summary>
+ /// Gets or sets the list of the merge strings used to merge tokens during encoding.
+ /// </summary>
+ public IEnumerable<string>? Merges { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional special tokens to use.
+ /// </summary>
+ public Dictionary<string, int>? SpecialTokens { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional normalizer to normalize the input text before encoding it.
+ /// </summary>
+ public Normalizer? Normalizer { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional pre-tokenizer to split the input text into tokens before encoding it.
+ /// </summary>
+ public PreTokenizer? PreTokenizer { get; set; }
+
+ /// <summary>
+ /// Gets or sets the Unknown token.
+ /// </summary>
+ public string? UnknownToken { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value indicating whether to merge the sequence of the unknown tokens together.
+ /// </summary>
+ public bool FuseUnknownTokens { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional prefix to be used for every subword that is not a beginning-of-word token
+ /// </summary>
+ public string? ContinuingSubwordPrefix { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional suffix to characterize the end-of-word and sub-word
+ /// </summary>
+ public string? EndOfWordSuffix { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value indicating whether to handle the input text in byte level.
+ /// if true, the input text will be converted to UTF-8 bytes before encoding it.
+ /// Additionally, some ASCII characters will be transformed to different characters (e.g Space character will be transformed to 'Ġ' character).
+ /// </summary>
+ public bool ByteLevel { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional beginning of sentence token to be used when encoding the input text.
+ /// </summary>
+ /// <remarks>
+ /// When specified, this token will be added to the beginning of the input text before encoding it.
+ /// This is useful for models that require a specific token to indicate the start of a sentence.
+ /// This token should be present in the vocabulary.
+ /// </remarks>
+ public string? BeginningOfSentenceToken { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional end of sentence token to be used when encoding the input text.
+ /// </summary>
+ /// <remarks>
+ /// When specified, this token will be added to the end of the input text before encoding it.
+ /// This is useful for models that require a specific token to indicate the end of a sentence.
+ /// This token should be present in the vocabulary.
+ /// </remarks>
+ public string? EndOfSentenceToken { get; set; }
+ }
+}

src/Microsoft.ML.Tokenizers/Model/CodeGenTokenizer.cs

@@ -1263,7 +1263,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(BeginningOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(BeginningOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1272,7 +1272,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(EndOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(EndOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1281,7 +1281,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(UnknownToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(UnknownToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1306,7 +1306,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  ReadOnlySpan<char> span = firstToken && hasPrefixSpace && s.Length > 0 && s[0] == _transformedSpace ? s.AsSpan(1) : s.AsSpan();
  firstToken = false;
- AppendToBytesArray(span, ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(span, ref bytes, ref bytesIndex);
  }
  }
 
@@ -1564,27 +1564,6 @@ public OperationStatus Decode(IEnumerable<int> ids, Span<char> destination, bool
  return null;
  }
 
- private void AppendToBytesArray(ReadOnlySpan<char> text, ref byte[] bytes, ref int bytesIndex)
- {
- IReadOnlyDictionary<char, char> unicodeToByte = ByteToUnicodeEncoding.Instance.UnicodeToByte;
- for (int i = 0; i < text.Length; i++)
- {
- if (unicodeToByte.TryGetValue(text[i], out char c))
- {
- if (bytesIndex >= bytes.Length)
- {
- Helpers.ArrayPoolGrow<byte>(ref bytes, bytes.Length * 2);
- }
-
- bytes[bytesIndex++] = (byte)c;
- continue;
- }
-
- // rare cases
- i += Helpers.EncodeCodePointToUtf8(text, i, ref bytes, ref bytesIndex) - 1;
- }
- }
-
  //
  // Private & Internal methods
  //

src/Microsoft.ML.Tokenizers/Model/Word.cs

@@ -289,15 +289,31 @@ public override string ToString()
  return sb.ToString();
  }
 
- public void ToTokens(SortedDictionary<int, string> vocabReverse, List<EncodedToken> tokens, int offset)
+ public void ToTokens(SortedDictionary<int, string> vocabReverse, List<EncodedToken> tokens, int offset, ReadOnlySpan<int> mapping)
  {
  int index = 0;
 
- for (int i = 0; i < SymbolsCount; i++)
+ if (mapping.IsEmpty)
+ {
+ for (int i = 0; i < SymbolsCount; i++)
+ {
+ int endIndex = index + _symbols[i].Len;
+ tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(index + offset, index + offset + _symbols[i].Len)));
+ index += _symbols[i].Len;
+ }
+ }
+ else
  {
- int endIndex = index + _symbols[i].Len;
- tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(index + offset, index + offset + _symbols[i].Len)));
- index += _symbols[i].Len;
+ for (int i = 0; i < SymbolsCount; i++)
+ {
+ int endIndex = index + _symbols[i].Len;
+
+ int mappedIndex = mapping[index];
+ int mappedEndIndex = mapping[endIndex - 1] + 1;
+
+ tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(mappedIndex + offset, mappedEndIndex + offset)));
+ index += _symbols[i].Len;
+ }
  }
  }
  }

eng/Versions.props

@@ -100,7 +100,7 @@
  <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
  <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
  <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
- <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25126.1</MicrosoftMLTestTokenizersVersion>
+ <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25161.1</MicrosoftMLTestTokenizersVersion>
  <SystemDataSqlClientVersion>4.9.0</SystemDataSqlClientVersion>
  <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
  <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>

src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs

@@ -0,0 +1,100 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+
+namespace Microsoft.ML.Tokenizers
+{
+ /// <summary>
+ /// Options for the BPE tokenizer.
+ /// </summary>
+ public sealed class BpeOptions
+ {
+ /// <summary>
+ /// Initializes a new instance of the <see cref="BpeOptions"/> class.
+ /// </summary>
+ public BpeOptions(IEnumerable<(string Token, int Id)> vocabulary)
+ {
+ if (vocabulary == null)
+ {
+ throw new ArgumentNullException(nameof(vocabulary));
+ }
+
+ Vocabulary = vocabulary;
+ }
+
+ /// <summary>
+ /// Gets or sets the vocabulary to use.
+ /// </summary>
+ public IEnumerable<(string Token, int Id)> Vocabulary { get; }
+
+ /// <summary>
+ /// Gets or sets the list of the merge strings used to merge tokens during encoding.
+ /// </summary>
+ public IEnumerable<string>? Merges { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional special tokens to use.
+ /// </summary>
+ public Dictionary<string, int>? SpecialTokens { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional normalizer to normalize the input text before encoding it.
+ /// </summary>
+ public Normalizer? Normalizer { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional pre-tokenizer to split the input text into tokens before encoding it.
+ /// </summary>
+ public PreTokenizer? PreTokenizer { get; set; }
+
+ /// <summary>
+ /// Gets or sets the Unknown token.
+ /// </summary>
+ public string? UnknownToken { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value indicating whether to merge the sequence of the unknown tokens together.
+ /// </summary>
+ public bool FuseUnknownTokens { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional prefix to be used for every subword that is not a beginning-of-word token
+ /// </summary>
+ public string? ContinuingSubwordPrefix { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional suffix to characterize the end-of-word and sub-word
+ /// </summary>
+ public string? EndOfWordSuffix { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value indicating whether to handle the input text in byte level.
+ /// if true, the input text will be converted to UTF-8 bytes before encoding it.
+ /// Additionally, some ASCII characters will be transformed to different characters (e.g Space character will be transformed to 'Ġ' character).
+ /// </summary>
+ public bool ByteLevel { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional beginning of sentence token to be used when encoding the input text.
+ /// </summary>
+ /// <remarks>
+ /// When specified, this token will be added to the beginning of the input text before encoding it.
+ /// This is useful for models that require a specific token to indicate the start of a sentence.
+ /// This token should be present in the vocabulary.
+ /// </remarks>
+ public string? BeginningOfSentenceToken { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional end of sentence token to be used when encoding the input text.
+ /// </summary>
+ /// <remarks>
+ /// When specified, this token will be added to the end of the input text before encoding it.
+ /// This is useful for models that require a specific token to indicate the end of a sentence.
+ /// This token should be present in the vocabulary.
+ /// </remarks>
+ public string? EndOfSentenceToken { get; set; }
+ }
+}

src/Microsoft.ML.Tokenizers/Model/CodeGenTokenizer.cs

@@ -1263,7 +1263,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(BeginningOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(BeginningOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1272,7 +1272,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(EndOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(EndOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1281,7 +1281,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(UnknownToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(UnknownToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1306,7 +1306,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  ReadOnlySpan<char> span = firstToken && hasPrefixSpace && s.Length > 0 && s[0] == _transformedSpace ? s.AsSpan(1) : s.AsSpan();
  firstToken = false;
- AppendToBytesArray(span, ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(span, ref bytes, ref bytesIndex);
  }
  }
 
@@ -1564,27 +1564,6 @@ public OperationStatus Decode(IEnumerable<int> ids, Span<char> destination, bool
  return null;
  }
 
- private void AppendToBytesArray(ReadOnlySpan<char> text, ref byte[] bytes, ref int bytesIndex)
- {
- IReadOnlyDictionary<char, char> unicodeToByte = ByteToUnicodeEncoding.Instance.UnicodeToByte;
- for (int i = 0; i < text.Length; i++)
- {
- if (unicodeToByte.TryGetValue(text[i], out char c))
- {
- if (bytesIndex >= bytes.Length)
- {
- Helpers.ArrayPoolGrow<byte>(ref bytes, bytes.Length * 2);
- }
-
- bytes[bytesIndex++] = (byte)c;
- continue;
- }
-
- // rare cases
- i += Helpers.EncodeCodePointToUtf8(text, i, ref bytes, ref bytesIndex) - 1;
- }
- }
-
  //
  // Private & Internal methods
  //

src/Microsoft.ML.Tokenizers/Model/Word.cs

@@ -289,15 +289,31 @@ public override string ToString()
  return sb.ToString();
  }
 
- public void ToTokens(SortedDictionary<int, string> vocabReverse, List<EncodedToken> tokens, int offset)
+ public void ToTokens(SortedDictionary<int, string> vocabReverse, List<EncodedToken> tokens, int offset, ReadOnlySpan<int> mapping)
  {
  int index = 0;
 
- for (int i = 0; i < SymbolsCount; i++)
+ if (mapping.IsEmpty)
+ {
+ for (int i = 0; i < SymbolsCount; i++)
+ {
+ int endIndex = index + _symbols[i].Len;
+ tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(index + offset, index + offset + _symbols[i].Len)));
+ index += _symbols[i].Len;
+ }
+ }
+ else
  {
- int endIndex = index + _symbols[i].Len;
- tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(index + offset, index + offset + _symbols[i].Len)));
- index += _symbols[i].Len;
+ for (int i = 0; i < SymbolsCount; i++)
+ {
+ int endIndex = index + _symbols[i].Len;
+
+ int mappedIndex = mapping[index];
+ int mappedEndIndex = mapping[endIndex - 1] + 1;
+
+ tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(mappedIndex + offset, mappedEndIndex + offset)));
+ index += _symbols[i].Len;
+ }
  }
  }
  }

eng/Versions.props

@@ -100,7 +100,7 @@
  <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
  <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
  <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
- <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25126.1</MicrosoftMLTestTokenizersVersion>
+ <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25161.1</MicrosoftMLTestTokenizersVersion>
  <SystemDataSqlClientVersion>4.9.0</SystemDataSqlClientVersion>
  <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
  <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>

src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs

@@ -0,0 +1,100 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+
+namespace Microsoft.ML.Tokenizers
+{
+ /// <summary>
+ /// Options for the BPE tokenizer.
+ /// </summary>
+ public sealed class BpeOptions
+ {
+ /// <summary>
+ /// Initializes a new instance of the <see cref="BpeOptions"/> class.
+ /// </summary>
+ public BpeOptions(IEnumerable<(string Token, int Id)> vocabulary)
+ {
+ if (vocabulary == null)
+ {
+ throw new ArgumentNullException(nameof(vocabulary));
+ }
+
+ Vocabulary = vocabulary;
+ }
+
+ /// <summary>
+ /// Gets or sets the vocabulary to use.
+ /// </summary>
+ public IEnumerable<(string Token, int Id)> Vocabulary { get; }
+
+ /// <summary>
+ /// Gets or sets the list of the merge strings used to merge tokens during encoding.
+ /// </summary>
+ public IEnumerable<string>? Merges { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional special tokens to use.
+ /// </summary>
+ public Dictionary<string, int>? SpecialTokens { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional normalizer to normalize the input text before encoding it.
+ /// </summary>
+ public Normalizer? Normalizer { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional pre-tokenizer to split the input text into tokens before encoding it.
+ /// </summary>
+ public PreTokenizer? PreTokenizer { get; set; }
+
+ /// <summary>
+ /// Gets or sets the Unknown token.
+ /// </summary>
+ public string? UnknownToken { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value indicating whether to merge the sequence of the unknown tokens together.
+ /// </summary>
+ public bool FuseUnknownTokens { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional prefix to be used for every subword that is not a beginning-of-word token
+ /// </summary>
+ public string? ContinuingSubwordPrefix { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional suffix to characterize the end-of-word and sub-word
+ /// </summary>
+ public string? EndOfWordSuffix { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value indicating whether to handle the input text in byte level.
+ /// if true, the input text will be converted to UTF-8 bytes before encoding it.
+ /// Additionally, some ASCII characters will be transformed to different characters (e.g Space character will be transformed to 'Ġ' character).
+ /// </summary>
+ public bool ByteLevel { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional beginning of sentence token to be used when encoding the input text.
+ /// </summary>
+ /// <remarks>
+ /// When specified, this token will be added to the beginning of the input text before encoding it.
+ /// This is useful for models that require a specific token to indicate the start of a sentence.
+ /// This token should be present in the vocabulary.
+ /// </remarks>
+ public string? BeginningOfSentenceToken { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional end of sentence token to be used when encoding the input text.
+ /// </summary>
+ /// <remarks>
+ /// When specified, this token will be added to the end of the input text before encoding it.
+ /// This is useful for models that require a specific token to indicate the end of a sentence.
+ /// This token should be present in the vocabulary.
+ /// </remarks>
+ public string? EndOfSentenceToken { get; set; }
+ }
+}

src/Microsoft.ML.Tokenizers/Model/CodeGenTokenizer.cs

@@ -1263,7 +1263,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(BeginningOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(BeginningOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1272,7 +1272,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(EndOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(EndOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1281,7 +1281,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(UnknownToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(UnknownToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1306,7 +1306,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  ReadOnlySpan<char> span = firstToken && hasPrefixSpace && s.Length > 0 && s[0] == _transformedSpace ? s.AsSpan(1) : s.AsSpan();
  firstToken = false;
- AppendToBytesArray(span, ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(span, ref bytes, ref bytesIndex);
  }
  }
 
@@ -1564,27 +1564,6 @@ public OperationStatus Decode(IEnumerable<int> ids, Span<char> destination, bool
  return null;
  }
 
- private void AppendToBytesArray(ReadOnlySpan<char> text, ref byte[] bytes, ref int bytesIndex)
- {
- IReadOnlyDictionary<char, char> unicodeToByte = ByteToUnicodeEncoding.Instance.UnicodeToByte;
- for (int i = 0; i < text.Length; i++)
- {
- if (unicodeToByte.TryGetValue(text[i], out char c))
- {
- if (bytesIndex >= bytes.Length)
- {
- Helpers.ArrayPoolGrow<byte>(ref bytes, bytes.Length * 2);
- }
-
- bytes[bytesIndex++] = (byte)c;
- continue;
- }
-
- // rare cases
- i += Helpers.EncodeCodePointToUtf8(text, i, ref bytes, ref bytesIndex) - 1;
- }
- }
-
  //
  // Private & Internal methods
  //

src/Microsoft.ML.Tokenizers/Model/Word.cs

@@ -289,15 +289,31 @@ public override string ToString()
  return sb.ToString();
  }
 
- public void ToTokens(SortedDictionary<int, string> vocabReverse, List<EncodedToken> tokens, int offset)
+ public void ToTokens(SortedDictionary<int, string> vocabReverse, List<EncodedToken> tokens, int offset, ReadOnlySpan<int> mapping)
  {
  int index = 0;
 
- for (int i = 0; i < SymbolsCount; i++)
+ if (mapping.IsEmpty)
+ {
+ for (int i = 0; i < SymbolsCount; i++)
+ {
+ int endIndex = index + _symbols[i].Len;
+ tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(index + offset, index + offset + _symbols[i].Len)));
+ index += _symbols[i].Len;
+ }
+ }
+ else
  {
- int endIndex = index + _symbols[i].Len;
- tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(index + offset, index + offset + _symbols[i].Len)));
- index += _symbols[i].Len;
+ for (int i = 0; i < SymbolsCount; i++)
+ {
+ int endIndex = index + _symbols[i].Len;
+
+ int mappedIndex = mapping[index];
+ int mappedEndIndex = mapping[endIndex - 1] + 1;
+
+ tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(mappedIndex + offset, mappedEndIndex + offset)));
+ index += _symbols[i].Len;
+ }
  }
  }
  }

eng/Versions.props

@@ -100,7 +100,7 @@
  <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
  <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
  <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
- <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25126.1</MicrosoftMLTestTokenizersVersion>
+ <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25161.1</MicrosoftMLTestTokenizersVersion>
  <SystemDataSqlClientVersion>4.9.0</SystemDataSqlClientVersion>
  <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
  <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>

src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs

@@ -0,0 +1,100 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+
+namespace Microsoft.ML.Tokenizers
+{
+ /// <summary>
+ /// Options for the BPE tokenizer.
+ /// </summary>
+ public sealed class BpeOptions
+ {
+ /// <summary>
+ /// Initializes a new instance of the <see cref="BpeOptions"/> class.
+ /// </summary>
+ public BpeOptions(IEnumerable<(string Token, int Id)> vocabulary)
+ {
+ if (vocabulary == null)
+ {
+ throw new ArgumentNullException(nameof(vocabulary));
+ }
+
+ Vocabulary = vocabulary;
+ }
+
+ /// <summary>
+ /// Gets or sets the vocabulary to use.
+ /// </summary>
+ public IEnumerable<(string Token, int Id)> Vocabulary { get; }
+
+ /// <summary>
+ /// Gets or sets the list of the merge strings used to merge tokens during encoding.
+ /// </summary>
+ public IEnumerable<string>? Merges { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional special tokens to use.
+ /// </summary>
+ public Dictionary<string, int>? SpecialTokens { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional normalizer to normalize the input text before encoding it.
+ /// </summary>
+ public Normalizer? Normalizer { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional pre-tokenizer to split the input text into tokens before encoding it.
+ /// </summary>
+ public PreTokenizer? PreTokenizer { get; set; }
+
+ /// <summary>
+ /// Gets or sets the Unknown token.
+ /// </summary>
+ public string? UnknownToken { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value indicating whether to merge the sequence of the unknown tokens together.
+ /// </summary>
+ public bool FuseUnknownTokens { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional prefix to be used for every subword that is not a beginning-of-word token
+ /// </summary>
+ public string? ContinuingSubwordPrefix { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional suffix to characterize the end-of-word and sub-word
+ /// </summary>
+ public string? EndOfWordSuffix { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value indicating whether to handle the input text in byte level.
+ /// if true, the input text will be converted to UTF-8 bytes before encoding it.
+ /// Additionally, some ASCII characters will be transformed to different characters (e.g Space character will be transformed to 'Ġ' character).
+ /// </summary>
+ public bool ByteLevel { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional beginning of sentence token to be used when encoding the input text.
+ /// </summary>
+ /// <remarks>
+ /// When specified, this token will be added to the beginning of the input text before encoding it.
+ /// This is useful for models that require a specific token to indicate the start of a sentence.
+ /// This token should be present in the vocabulary.
+ /// </remarks>
+ public string? BeginningOfSentenceToken { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional end of sentence token to be used when encoding the input text.
+ /// </summary>
+ /// <remarks>
+ /// When specified, this token will be added to the end of the input text before encoding it.
+ /// This is useful for models that require a specific token to indicate the end of a sentence.
+ /// This token should be present in the vocabulary.
+ /// </remarks>
+ public string? EndOfSentenceToken { get; set; }
+ }
+}

src/Microsoft.ML.Tokenizers/Model/CodeGenTokenizer.cs

@@ -1263,7 +1263,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(BeginningOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(BeginningOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1272,7 +1272,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(EndOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(EndOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1281,7 +1281,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(UnknownToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(UnknownToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1306,7 +1306,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  ReadOnlySpan<char> span = firstToken && hasPrefixSpace && s.Length > 0 && s[0] == _transformedSpace ? s.AsSpan(1) : s.AsSpan();
  firstToken = false;
- AppendToBytesArray(span, ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(span, ref bytes, ref bytesIndex);
  }
  }
 
@@ -1564,27 +1564,6 @@ public OperationStatus Decode(IEnumerable<int> ids, Span<char> destination, bool
  return null;
  }
 
- private void AppendToBytesArray(ReadOnlySpan<char> text, ref byte[] bytes, ref int bytesIndex)
- {
- IReadOnlyDictionary<char, char> unicodeToByte = ByteToUnicodeEncoding.Instance.UnicodeToByte;
- for (int i = 0; i < text.Length; i++)
- {
- if (unicodeToByte.TryGetValue(text[i], out char c))
- {
- if (bytesIndex >= bytes.Length)
- {
- Helpers.ArrayPoolGrow<byte>(ref bytes, bytes.Length * 2);
- }
-
- bytes[bytesIndex++] = (byte)c;
- continue;
- }
-
- // rare cases
- i += Helpers.EncodeCodePointToUtf8(text, i, ref bytes, ref bytesIndex) - 1;
- }
- }
-
  //
  // Private & Internal methods
  //

src/Microsoft.ML.Tokenizers/Model/Word.cs

@@ -289,15 +289,31 @@ public override string ToString()
  return sb.ToString();
  }
 
- public void ToTokens(SortedDictionary<int, string> vocabReverse, List<EncodedToken> tokens, int offset)
+ public void ToTokens(SortedDictionary<int, string> vocabReverse, List<EncodedToken> tokens, int offset, ReadOnlySpan<int> mapping)
  {
  int index = 0;
 
- for (int i = 0; i < SymbolsCount; i++)
+ if (mapping.IsEmpty)
+ {
+ for (int i = 0; i < SymbolsCount; i++)
+ {
+ int endIndex = index + _symbols[i].Len;
+ tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(index + offset, index + offset + _symbols[i].Len)));
+ index += _symbols[i].Len;
+ }
+ }
+ else
  {
- int endIndex = index + _symbols[i].Len;
- tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(index + offset, index + offset + _symbols[i].Len)));
- index += _symbols[i].Len;
+ for (int i = 0; i < SymbolsCount; i++)
+ {
+ int endIndex = index + _symbols[i].Len;
+
+ int mappedIndex = mapping[index];
+ int mappedEndIndex = mapping[endIndex - 1] + 1;
+
+ tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(mappedIndex + offset, mappedEndIndex + offset)));
+ index += _symbols[i].Len;
+ }
  }
  }
  }

eng/Versions.props

@@ -100,7 +100,7 @@
  <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
  <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
  <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
- <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25126.1</MicrosoftMLTestTokenizersVersion>
+ <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25161.1</MicrosoftMLTestTokenizersVersion>
  <SystemDataSqlClientVersion>4.9.0</SystemDataSqlClientVersion>
  <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
  <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>

src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs

@@ -0,0 +1,100 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+
+namespace Microsoft.ML.Tokenizers
+{
+ /// <summary>
+ /// Options for the BPE tokenizer.
+ /// </summary>
+ public sealed class BpeOptions
+ {
+ /// <summary>
+ /// Initializes a new instance of the <see cref="BpeOptions"/> class.
+ /// </summary>
+ public BpeOptions(IEnumerable<(string Token, int Id)> vocabulary)
+ {
+ if (vocabulary == null)
+ {
+ throw new ArgumentNullException(nameof(vocabulary));
+ }
+
+ Vocabulary = vocabulary;
+ }
+
+ /// <summary>
+ /// Gets or sets the vocabulary to use.
+ /// </summary>
+ public IEnumerable<(string Token, int Id)> Vocabulary { get; }
+
+ /// <summary>
+ /// Gets or sets the list of the merge strings used to merge tokens during encoding.
+ /// </summary>
+ public IEnumerable<string>? Merges { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional special tokens to use.
+ /// </summary>
+ public Dictionary<string, int>? SpecialTokens { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional normalizer to normalize the input text before encoding it.
+ /// </summary>
+ public Normalizer? Normalizer { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional pre-tokenizer to split the input text into tokens before encoding it.
+ /// </summary>
+ public PreTokenizer? PreTokenizer { get; set; }
+
+ /// <summary>
+ /// Gets or sets the Unknown token.
+ /// </summary>
+ public string? UnknownToken { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value indicating whether to merge the sequence of the unknown tokens together.
+ /// </summary>
+ public bool FuseUnknownTokens { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional prefix to be used for every subword that is not a beginning-of-word token
+ /// </summary>
+ public string? ContinuingSubwordPrefix { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional suffix to characterize the end-of-word and sub-word
+ /// </summary>
+ public string? EndOfWordSuffix { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value indicating whether to handle the input text in byte level.
+ /// if true, the input text will be converted to UTF-8 bytes before encoding it.
+ /// Additionally, some ASCII characters will be transformed to different characters (e.g Space character will be transformed to 'Ġ' character).
+ /// </summary>
+ public bool ByteLevel { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional beginning of sentence token to be used when encoding the input text.
+ /// </summary>
+ /// <remarks>
+ /// When specified, this token will be added to the beginning of the input text before encoding it.
+ /// This is useful for models that require a specific token to indicate the start of a sentence.
+ /// This token should be present in the vocabulary.
+ /// </remarks>
+ public string? BeginningOfSentenceToken { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional end of sentence token to be used when encoding the input text.
+ /// </summary>
+ /// <remarks>
+ /// When specified, this token will be added to the end of the input text before encoding it.
+ /// This is useful for models that require a specific token to indicate the end of a sentence.
+ /// This token should be present in the vocabulary.
+ /// </remarks>
+ public string? EndOfSentenceToken { get; set; }
+ }
+}

src/Microsoft.ML.Tokenizers/Model/CodeGenTokenizer.cs

@@ -1263,7 +1263,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(BeginningOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(BeginningOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1272,7 +1272,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(EndOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(EndOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1281,7 +1281,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(UnknownToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(UnknownToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1306,7 +1306,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  ReadOnlySpan<char> span = firstToken && hasPrefixSpace && s.Length > 0 && s[0] == _transformedSpace ? s.AsSpan(1) : s.AsSpan();
  firstToken = false;
- AppendToBytesArray(span, ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(span, ref bytes, ref bytesIndex);
  }
  }
 
@@ -1564,27 +1564,6 @@ public OperationStatus Decode(IEnumerable<int> ids, Span<char> destination, bool
  return null;
  }
 
- private void AppendToBytesArray(ReadOnlySpan<char> text, ref byte[] bytes, ref int bytesIndex)
- {
- IReadOnlyDictionary<char, char> unicodeToByte = ByteToUnicodeEncoding.Instance.UnicodeToByte;
- for (int i = 0; i < text.Length; i++)
- {
- if (unicodeToByte.TryGetValue(text[i], out char c))
- {
- if (bytesIndex >= bytes.Length)
- {
- Helpers.ArrayPoolGrow<byte>(ref bytes, bytes.Length * 2);
- }
-
- bytes[bytesIndex++] = (byte)c;
- continue;
- }
-
- // rare cases
- i += Helpers.EncodeCodePointToUtf8(text, i, ref bytes, ref bytesIndex) - 1;
- }
- }
-
  //
  // Private & Internal methods
  //

src/Microsoft.ML.Tokenizers/Model/Word.cs

@@ -289,15 +289,31 @@ public override string ToString()
  return sb.ToString();
  }
 
- public void ToTokens(SortedDictionary<int, string> vocabReverse, List<EncodedToken> tokens, int offset)
+ public void ToTokens(SortedDictionary<int, string> vocabReverse, List<EncodedToken> tokens, int offset, ReadOnlySpan<int> mapping)
  {
  int index = 0;
 
- for (int i = 0; i < SymbolsCount; i++)
+ if (mapping.IsEmpty)
+ {
+ for (int i = 0; i < SymbolsCount; i++)
+ {
+ int endIndex = index + _symbols[i].Len;
+ tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(index + offset, index + offset + _symbols[i].Len)));
+ index += _symbols[i].Len;
+ }
+ }
+ else
  {
- int endIndex = index + _symbols[i].Len;
- tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(index + offset, index + offset + _symbols[i].Len)));
- index += _symbols[i].Len;
+ for (int i = 0; i < SymbolsCount; i++)
+ {
+ int endIndex = index + _symbols[i].Len;
+
+ int mappedIndex = mapping[index];
+ int mappedEndIndex = mapping[endIndex - 1] + 1;
+
+ tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(mappedIndex + offset, mappedEndIndex + offset)));
+ index += _symbols[i].Len;
+ }
  }
  }
  }

eng/Versions.props

@@ -100,7 +100,7 @@
  <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
  <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
  <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
- <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25126.1</MicrosoftMLTestTokenizersVersion>
+ <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25161.1</MicrosoftMLTestTokenizersVersion>
  <SystemDataSqlClientVersion>4.9.0</SystemDataSqlClientVersion>
  <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
  <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>

src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs

@@ -0,0 +1,100 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+
+namespace Microsoft.ML.Tokenizers
+{
+ /// <summary>
+ /// Options for the BPE tokenizer.
+ /// </summary>
+ public sealed class BpeOptions
+ {
+ /// <summary>
+ /// Initializes a new instance of the <see cref="BpeOptions"/> class.
+ /// </summary>
+ public BpeOptions(IEnumerable<(string Token, int Id)> vocabulary)
+ {
+ if (vocabulary == null)
+ {
+ throw new ArgumentNullException(nameof(vocabulary));
+ }
+
+ Vocabulary = vocabulary;
+ }
+
+ /// <summary>
+ /// Gets or sets the vocabulary to use.
+ /// </summary>
+ public IEnumerable<(string Token, int Id)> Vocabulary { get; }
+
+ /// <summary>
+ /// Gets or sets the list of the merge strings used to merge tokens during encoding.
+ /// </summary>
+ public IEnumerable<string>? Merges { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional special tokens to use.
+ /// </summary>
+ public Dictionary<string, int>? SpecialTokens { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional normalizer to normalize the input text before encoding it.
+ /// </summary>
+ public Normalizer? Normalizer { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional pre-tokenizer to split the input text into tokens before encoding it.
+ /// </summary>
+ public PreTokenizer? PreTokenizer { get; set; }
+
+ /// <summary>
+ /// Gets or sets the Unknown token.
+ /// </summary>
+ public string? UnknownToken { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value indicating whether to merge the sequence of the unknown tokens together.
+ /// </summary>
+ public bool FuseUnknownTokens { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional prefix to be used for every subword that is not a beginning-of-word token
+ /// </summary>
+ public string? ContinuingSubwordPrefix { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional suffix to characterize the end-of-word and sub-word
+ /// </summary>
+ public string? EndOfWordSuffix { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value indicating whether to handle the input text in byte level.
+ /// if true, the input text will be converted to UTF-8 bytes before encoding it.
+ /// Additionally, some ASCII characters will be transformed to different characters (e.g Space character will be transformed to 'Ġ' character).
+ /// </summary>
+ public bool ByteLevel { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional beginning of sentence token to be used when encoding the input text.
+ /// </summary>
+ /// <remarks>
+ /// When specified, this token will be added to the beginning of the input text before encoding it.
+ /// This is useful for models that require a specific token to indicate the start of a sentence.
+ /// This token should be present in the vocabulary.
+ /// </remarks>
+ public string? BeginningOfSentenceToken { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional end of sentence token to be used when encoding the input text.
+ /// </summary>
+ /// <remarks>
+ /// When specified, this token will be added to the end of the input text before encoding it.
+ /// This is useful for models that require a specific token to indicate the end of a sentence.
+ /// This token should be present in the vocabulary.
+ /// </remarks>
+ public string? EndOfSentenceToken { get; set; }
+ }
+}

src/Microsoft.ML.Tokenizers/Model/CodeGenTokenizer.cs

@@ -1263,7 +1263,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(BeginningOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(BeginningOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1272,7 +1272,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(EndOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(EndOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1281,7 +1281,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(UnknownToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(UnknownToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1306,7 +1306,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  ReadOnlySpan<char> span = firstToken && hasPrefixSpace && s.Length > 0 && s[0] == _transformedSpace ? s.AsSpan(1) : s.AsSpan();
  firstToken = false;
- AppendToBytesArray(span, ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(span, ref bytes, ref bytesIndex);
  }
  }
 
@@ -1564,27 +1564,6 @@ public OperationStatus Decode(IEnumerable<int> ids, Span<char> destination, bool
  return null;
  }
 
- private void AppendToBytesArray(ReadOnlySpan<char> text, ref byte[] bytes, ref int bytesIndex)
- {
- IReadOnlyDictionary<char, char> unicodeToByte = ByteToUnicodeEncoding.Instance.UnicodeToByte;
- for (int i = 0; i < text.Length; i++)
- {
- if (unicodeToByte.TryGetValue(text[i], out char c))
- {
- if (bytesIndex >= bytes.Length)
- {
- Helpers.ArrayPoolGrow<byte>(ref bytes, bytes.Length * 2);
- }
-
- bytes[bytesIndex++] = (byte)c;
- continue;
- }
-
- // rare cases
- i += Helpers.EncodeCodePointToUtf8(text, i, ref bytes, ref bytesIndex) - 1;
- }
- }
-
  //
  // Private & Internal methods
  //

src/Microsoft.ML.Tokenizers/Model/Word.cs

@@ -289,15 +289,31 @@ public override string ToString()
  return sb.ToString();
  }
 
- public void ToTokens(SortedDictionary<int, string> vocabReverse, List<EncodedToken> tokens, int offset)
+ public void ToTokens(SortedDictionary<int, string> vocabReverse, List<EncodedToken> tokens, int offset, ReadOnlySpan<int> mapping)
  {
  int index = 0;
 
- for (int i = 0; i < SymbolsCount; i++)
+ if (mapping.IsEmpty)
+ {
+ for (int i = 0; i < SymbolsCount; i++)
+ {
+ int endIndex = index + _symbols[i].Len;
+ tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(index + offset, index + offset + _symbols[i].Len)));
+ index += _symbols[i].Len;
+ }
+ }
+ else
  {
- int endIndex = index + _symbols[i].Len;
- tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(index + offset, index + offset + _symbols[i].Len)));
- index += _symbols[i].Len;
+ for (int i = 0; i < SymbolsCount; i++)
+ {
+ int endIndex = index + _symbols[i].Len;
+
+ int mappedIndex = mapping[index];
+ int mappedEndIndex = mapping[endIndex - 1] + 1;
+
+ tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(mappedIndex + offset, mappedEndIndex + offset)));
+ index += _symbols[i].Len;
+ }
  }
  }
  }

eng/Versions.props

@@ -100,7 +100,7 @@
  <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
  <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
  <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
- <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25126.1</MicrosoftMLTestTokenizersVersion>
+ <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25161.1</MicrosoftMLTestTokenizersVersion>
  <SystemDataSqlClientVersion>4.9.0</SystemDataSqlClientVersion>
  <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
  <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>

src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs

@@ -0,0 +1,100 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+
+namespace Microsoft.ML.Tokenizers
+{
+ /// <summary>
+ /// Options for the BPE tokenizer.
+ /// </summary>
+ public sealed class BpeOptions
+ {
+ /// <summary>
+ /// Initializes a new instance of the <see cref="BpeOptions"/> class.
+ /// </summary>
+ public BpeOptions(IEnumerable<(string Token, int Id)> vocabulary)
+ {
+ if (vocabulary == null)
+ {
+ throw new ArgumentNullException(nameof(vocabulary));
+ }
+
+ Vocabulary = vocabulary;
+ }
+
+ /// <summary>
+ /// Gets or sets the vocabulary to use.
+ /// </summary>
+ public IEnumerable<(string Token, int Id)> Vocabulary { get; }
+
+ /// <summary>
+ /// Gets or sets the list of the merge strings used to merge tokens during encoding.
+ /// </summary>
+ public IEnumerable<string>? Merges { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional special tokens to use.
+ /// </summary>
+ public Dictionary<string, int>? SpecialTokens { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional normalizer to normalize the input text before encoding it.
+ /// </summary>
+ public Normalizer? Normalizer { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional pre-tokenizer to split the input text into tokens before encoding it.
+ /// </summary>
+ public PreTokenizer? PreTokenizer { get; set; }
+
+ /// <summary>
+ /// Gets or sets the Unknown token.
+ /// </summary>
+ public string? UnknownToken { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value indicating whether to merge the sequence of the unknown tokens together.
+ /// </summary>
+ public bool FuseUnknownTokens { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional prefix to be used for every subword that is not a beginning-of-word token
+ /// </summary>
+ public string? ContinuingSubwordPrefix { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional suffix to characterize the end-of-word and sub-word
+ /// </summary>
+ public string? EndOfWordSuffix { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value indicating whether to handle the input text in byte level.
+ /// if true, the input text will be converted to UTF-8 bytes before encoding it.
+ /// Additionally, some ASCII characters will be transformed to different characters (e.g Space character will be transformed to 'Ġ' character).
+ /// </summary>
+ public bool ByteLevel { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional beginning of sentence token to be used when encoding the input text.
+ /// </summary>
+ /// <remarks>
+ /// When specified, this token will be added to the beginning of the input text before encoding it.
+ /// This is useful for models that require a specific token to indicate the start of a sentence.
+ /// This token should be present in the vocabulary.
+ /// </remarks>
+ public string? BeginningOfSentenceToken { get; set; }
+
+ /// <summary>
+ /// Gets or sets the optional end of sentence token to be used when encoding the input text.
+ /// </summary>
+ /// <remarks>
+ /// When specified, this token will be added to the end of the input text before encoding it.
+ /// This is useful for models that require a specific token to indicate the end of a sentence.
+ /// This token should be present in the vocabulary.
+ /// </remarks>
+ public string? EndOfSentenceToken { get; set; }
+ }
+}

src/Microsoft.ML.Tokenizers/Model/CodeGenTokenizer.cs

@@ -1263,7 +1263,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(BeginningOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(BeginningOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1272,7 +1272,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(EndOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(EndOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1281,7 +1281,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  if (considerSpecialTokens)
  {
- AppendToBytesArray(UnknownToken!.AsSpan(), ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(UnknownToken!.AsSpan(), ref bytes, ref bytesIndex);
  }
  continue;
  }
@@ -1306,7 +1306,7 @@ public string Decode(IEnumerable<int> ids, bool hasPrefixSpace, bool considerSpe
  {
  ReadOnlySpan<char> span = firstToken && hasPrefixSpace && s.Length > 0 && s[0] == _transformedSpace ? s.AsSpan(1) : s.AsSpan();
  firstToken = false;
- AppendToBytesArray(span, ref bytes, ref bytesIndex);
+Helpers.AppendToBytesArray(span, ref bytes, ref bytesIndex);
  }
  }
 
@@ -1564,27 +1564,6 @@ public OperationStatus Decode(IEnumerable<int> ids, Span<char> destination, bool
  return null;
  }
 
- private void AppendToBytesArray(ReadOnlySpan<char> text, ref byte[] bytes, ref int bytesIndex)
- {
- IReadOnlyDictionary<char, char> unicodeToByte = ByteToUnicodeEncoding.Instance.UnicodeToByte;
- for (int i = 0; i < text.Length; i++)
- {
- if (unicodeToByte.TryGetValue(text[i], out char c))
- {
- if (bytesIndex >= bytes.Length)
- {
- Helpers.ArrayPoolGrow<byte>(ref bytes, bytes.Length * 2);
- }
-
- bytes[bytesIndex++] = (byte)c;
- continue;
- }
-
- // rare cases
- i += Helpers.EncodeCodePointToUtf8(text, i, ref bytes, ref bytesIndex) - 1;
- }
- }
-
  //
  // Private & Internal methods
  //

src/Microsoft.ML.Tokenizers/Model/Word.cs

@@ -289,15 +289,31 @@ public override string ToString()
  return sb.ToString();
  }
 
- public void ToTokens(SortedDictionary<int, string> vocabReverse, List<EncodedToken> tokens, int offset)
+ public void ToTokens(SortedDictionary<int, string> vocabReverse, List<EncodedToken> tokens, int offset, ReadOnlySpan<int> mapping)
  {
  int index = 0;
 
- for (int i = 0; i < SymbolsCount; i++)
+ if (mapping.IsEmpty)
+ {
+ for (int i = 0; i < SymbolsCount; i++)
+ {
+ int endIndex = index + _symbols[i].Len;
+ tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(index + offset, index + offset + _symbols[i].Len)));
+ index += _symbols[i].Len;
+ }
+ }
+ else
  {
- int endIndex = index + _symbols[i].Len;
- tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(index + offset, index + offset + _symbols[i].Len)));
- index += _symbols[i].Len;
+ for (int i = 0; i < SymbolsCount; i++)
+ {
+ int endIndex = index + _symbols[i].Len;
+
+ int mappedIndex = mapping[index];
+ int mappedEndIndex = mapping[endIndex - 1] + 1;
+
+ tokens.Add(new EncodedToken(_symbols[i].C, vocabReverse[_symbols[i].C], new Range(mappedIndex + offset, mappedEndIndex + offset)));
+ index += _symbols[i].Len;
+ }
  }
  }
  }