Create SentencePieceTokenizer from options object

eng/Versions.props

@@ -100,7 +100,7 @@
  <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
  <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
  <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
- <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25110.1</MicrosoftMLTestTokenizersVersion>
+ <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25126.1</MicrosoftMLTestTokenizersVersion>
  <SystemDataSqlClientVersion>4.9.0</SystemDataSqlClientVersion>
  <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
  <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>

src/Microsoft.ML.Tokenizers/Model/SentencePieceBaseModel.cs

@@ -59,6 +59,67 @@ internal SentencePieceBaseModel(ModelProto modelProto, bool addBos = false, bool
  specialTokens);
  }
 
+ internal SentencePieceBaseModel(SentencePieceOptions options)
+ {
+ if (options is null)
+ {
+ throw new ArgumentNullException(nameof(options));
+ }
+
+ if (options.Vocabulary is null)
+ {
+ throw new ArgumentNullException(nameof(options.Vocabulary));
+ }
+
+ if (options.BeginningOfSentenceToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.BeginningOfSentenceToken));
+ }
+
+ if (options.EndOfSentenceToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.EndOfSentenceToken));
+ }
+
+ if (options.UnknownToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.UnknownToken));
+ }
+
+ AddBeginningOfSentence = options.AddBeginningOfSentence;
+ AddEndOfSentence = options.AddEndOfSentence;
+ BeginningOfSentenceToken = options.BeginningOfSentenceToken;
+ EndOfSentenceToken = options.EndOfSentenceToken;
+ UnknownToken = options.UnknownToken;
+ AddDummyPrefix = options.AddDummyPrefix;
+ EscapeWhiteSpaces = options.EscapeWhiteSpaces;
+ TreatWhitespaceAsSuffix = options.TreatWhitespaceAsSuffix;
+ ByteFallback = options.ByteFallback;
+ SpecialTokens = options.SpecialTokens;
+
+ if (SpecialTokens is not null && SpecialTokens.Count > 0)
+ {
+ InternalSpecialTokens = new Dictionary<StringSpanOrdinalKey, int>();
+ SpecialTokensReverse = new Dictionary<int, string>();
+
+ foreach (var item in SpecialTokens)
+ {
+ InternalSpecialTokens.Add(new StringSpanOrdinalKey(item.Key), item.Value);
+ SpecialTokensReverse.Add(item.Value, item.Key);
+ }
+
+ // We create this Regex object without a timeout, as we expect the match operation to complete in O(N) time complexity. Note that `specialTokens` are treated as constants after the tokenizer is created.
+ SpecialTokensRegex = new Regex(string.Join("|", SpecialTokens.Keys.Select(s => Regex.Escape(s))), RegexOptions.Compiled);
+ }
+
+ Normalizer = new SentencePieceNormalizer(
+ options.PrecompiledNormalizationData,
+ options.RemoveExtraWhiteSpaces,
+ options.AddDummyPrefix, options.EscapeWhiteSpaces,
+ options.TreatWhitespaceAsSuffix,
+ SpecialTokens);
+ }
+
  internal Regex? SpecialTokensRegex { get; }
 
  internal Dictionary<StringSpanOrdinalKey, int>? InternalSpecialTokens { get; }
@@ -91,11 +152,11 @@ internal SentencePieceBaseModel(ModelProto modelProto, bool addBos = false, bool
 
  public string UnknownToken { get; }
 
- public int BeginningOfSentenceId { get; }
+ public int BeginningOfSentenceId { get; set; }
 
- public int EndOfSentenceId { get; }
+ public int EndOfSentenceId { get; set; }
 
- public int UnknownId { get; }
+ public int UnknownId { get; set; }
 
  public SentencePieceNormalizer? Normalizer { get; }
 

src/Microsoft.ML.Tokenizers/Model/SentencePieceBpeModel.cs

@@ -41,6 +41,52 @@ internal SentencePieceBpeModel(ModelProto modelProto, bool addBos, bool addEos,
  OneByteUtf8EncodingMaxId = ByteCodeToIdOffset + 0x7F; // 0x7F is the maximum value of the one byte UTF-8 character.
  }
 
+ internal SentencePieceBpeModel(SentencePieceOptions options) : base(options)
+ {
+ if (options.PrecompiledNormalizationData is not null)
+ {
+ throw new NotSupportedException("Normalization data is not supported for SentencePieceBpeModel.");
+ }
+
+ Debug.Assert(options.Vocabulary is not null);
+
+ int id = 0;
+ foreach (var item in options.Vocabulary!)
+ {
+ _vocab.Add(new StringSpanOrdinalKey(item.Token), (id, item.Score, (byte)ModelProto.Types.SentencePiece.Types.Type.Normal));
+ _vocabReverse.Add(id++, item.Token);
+ }
+
+ if (options.ByteFallback)
+ {
+ if (!_vocab.TryGetValue("<0x00>", out (int Id, float Score, byte Type) value))
+ {
+ throw new ArgumentException("'ByteFallback' is enabled but the vocabulary must include a special token for each byte value (0-255) in the format <0xNN>, where NN represents the byte's hexadecimal value.");
+ }
+
+ ByteCodeToIdOffset = value.Id;
+ OneByteUtf8EncodingMaxId = ByteCodeToIdOffset + 0x7F; // 0x7F is the maximum value of the one byte UTF-8 character.
+ }
+
+ if (!_vocab.TryGetValue(options.UnknownToken, out (int Id, float Score, byte Type) unknownToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the unknown token '{options.UnknownToken}'.");
+ }
+ UnknownId = unknownToken.Id;
+
+ if (!_vocab.TryGetValue(options.BeginningOfSentenceToken, out (int Id, float Score, byte Type) beginOfSentenceToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the beginning of sentence token '{options.BeginningOfSentenceToken}'.");
+ }
+ BeginningOfSentenceId = beginOfSentenceToken.Id;
+
+ if (!_vocab.TryGetValue(options.EndOfSentenceToken, out (int Id, float Score, byte Type) endOfSentenceToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the end of sentence token '{options.EndOfSentenceToken}'.");
+ }
+ EndOfSentenceId = endOfSentenceToken.Id;
+ }
+
  public override IReadOnlyDictionary<string, int> Vocabulary
  {
  get

src/Microsoft.ML.Tokenizers/Model/SentencePieceOptions.cs

@@ -0,0 +1,118 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+
+namespace Microsoft.ML.Tokenizers
+{
+#pragma warning disable MSML_NoInstanceInitializers
+ /// <summary>
+ /// The type of the SentencePiece model.
+ /// </summary>
+ public enum SentencePieceModelType
+ {
+ /// <summary>
+ /// The model type is not defined.
+ /// </summary>
+ Undefined = 0,
+
+ /// <summary>
+ /// The model type is Byte Pair Encoding (Bpe) model.
+ /// </summary>
+ Bpe = 1,
+
+ /// <summary>
+ /// The model type is Unigram model.
+ /// </summary>
+ Unigram = 2,
+ }
+
+ /// <summary>
+ /// Options for the SentencePiece tokenizer.
+ /// </summary>
+ /// <remarks>
+ /// The options are used to configure the SentencePiece tokenizer. Serialization is not guaranteed for this type.
+ /// </remarks>
+ public sealed class SentencePieceOptions
+ {
+ /// <summary>
+ /// The type of the SentencePiece model.
+ /// </summary>
+ public SentencePieceModelType ModelType { get; set; }
+
+ /// <summary>
+ /// Determines whether the model uses a byte fallback strategy to encode unknown tokens as byte sequences.
+ /// </summary>
+ /// <remarks>
+ /// The vocabulary must include a special token for each byte value (0-255) in the format &lt;0xNN&gt;,
+ /// where NN represents the byte's hexadecimal value (e.g., &lt;0x41&gt; for byte value 65).
+ /// </remarks>
+ public bool ByteFallback { get; set; }
+
+ /// <summary>
+ /// Indicate emitting the prefix character e.g. U+2581 at the beginning of sentence token during the normalization and encoding.
+ /// </summary>
+ public bool AddDummyPrefix { get; set; }
+
+ /// <summary>
+ /// Indicate if the spaces should be replaced with character U+2581 during the normalization and encoding. Default value is `true`.
+ /// </summary>
+ public bool EscapeWhiteSpaces { get; set; } = true;
+
+ /// <summary>
+ /// Indicate emitting the character U+2581 at the end of the last sentence token instead beginning of sentence token during the normalization and encoding.
+ /// </summary>
+ public bool TreatWhitespaceAsSuffix { get; set; }
+
+ /// <summary>
+ /// Indicate removing extra white spaces from the original string during the normalization.
+ /// </summary>
+ public bool RemoveExtraWhiteSpaces { get; set; }
+
+ /// <summary>
+ /// Indicate emitting the beginning of sentence token during the encoding. Default value is `true`.
+ /// </summary>
+ public bool AddBeginningOfSentence { get; set; } = true;
+
+ /// <summary>
+ /// Indicate emitting the end of sentence token during the encoding.
+ /// </summary>
+ public bool AddEndOfSentence { get; set; }
+
+ /// <summary>
+ /// The beginning of sentence token. Default value is `&lt;s&gt;`.
+ /// </summary>
+ public string BeginningOfSentenceToken { get; set; } = "<s>";
+
+ /// <summary>
+ /// The end of sentence token. Default value is `&lt;/s&gt;`.
+ /// </summary>
+ public string EndOfSentenceToken { get; set; } = "</s>";
+
+ /// <summary>
+ /// The unknown token. Default value is `&lt;unk&gt;`.
+ /// </summary>
+ public string UnknownToken { get; set; } = "<unk>";
+
+ /// <summary>
+ /// The data used for string normalization.
+ /// </summary>
+ public byte[]? PrecompiledNormalizationData { get; set; }
+
+ /// <summary>
+ /// Represent the vocabulary.
+ /// The list should be sorted by token ID, with entries passed in the order that corresponds to their IDs. In other words,
+ /// the first entry in the list will be mapped to ID 0, the second entry to ID 1, the third to ID 2, and so on.
+ /// Each entry represents a token and its corresponding score.
+ /// </summary>
+ public IEnumerable<(string Token, float Score)>? Vocabulary { get; set; }
+
+ /// <summary>
+ /// The special tokens.
+ /// Special tokens remain intact during encoding and are not split into sub-tokens.
+ /// </summary>
+ public IReadOnlyDictionary<string, int>? SpecialTokens { get; set; }
+ }
+#pragma warning restore MSML_NoInstanceInitializers
+}

src/Microsoft.ML.Tokenizers/Model/SentencePieceTokenizer.cs

@@ -30,6 +30,16 @@ internal SentencePieceTokenizer(ModelProto modelProto, bool addBos, bool addEos,
  };
  }
 
+ internal SentencePieceTokenizer(SentencePieceOptions options)
+ {
+ _model = options.ModelType switch
+ {
+ SentencePieceModelType.Bpe => new SentencePieceBpeModel(options),
+ SentencePieceModelType.Unigram => new SentencePieceUnigramModel(options),
+ _ => throw new ArgumentException($"The model type '{options.ModelType}' is not supported.", nameof(options.ModelType))
+ };
+ }
+
  /// <summary>
  /// The special tokens.
  /// </summary>
@@ -457,5 +467,19 @@ public static SentencePieceTokenizer Create(
 
  return new SentencePieceTokenizer(modelProto, addBeginOfSentence, addEndOfSentence, specialTokens);
  }
+
+ /// <summary>
+ /// Creates an instance of SentencePieceTokenizer.
+ /// </summary>
+ /// <param name="options">The options to use for the sentence piece tokenizer.</param>
+ public static SentencePieceTokenizer Create(SentencePieceOptions options)
+ {
+ if (options is null)
+ {
+ throw new ArgumentNullException(nameof(options));
+ }
+
+ return new SentencePieceTokenizer(options);
+ }
  }
 }

eng/Versions.props

@@ -100,7 +100,7 @@
  <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
  <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
  <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
- <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25110.1</MicrosoftMLTestTokenizersVersion>
+ <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25126.1</MicrosoftMLTestTokenizersVersion>
  <SystemDataSqlClientVersion>4.9.0</SystemDataSqlClientVersion>
  <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
  <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>

src/Microsoft.ML.Tokenizers/Model/SentencePieceBaseModel.cs

@@ -59,6 +59,67 @@ internal SentencePieceBaseModel(ModelProto modelProto, bool addBos = false, bool
  specialTokens);
  }
 
+ internal SentencePieceBaseModel(SentencePieceOptions options)
+ {
+ if (options is null)
+ {
+ throw new ArgumentNullException(nameof(options));
+ }
+
+ if (options.Vocabulary is null)
+ {
+ throw new ArgumentNullException(nameof(options.Vocabulary));
+ }
+
+ if (options.BeginningOfSentenceToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.BeginningOfSentenceToken));
+ }
+
+ if (options.EndOfSentenceToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.EndOfSentenceToken));
+ }
+
+ if (options.UnknownToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.UnknownToken));
+ }
+
+ AddBeginningOfSentence = options.AddBeginningOfSentence;
+ AddEndOfSentence = options.AddEndOfSentence;
+ BeginningOfSentenceToken = options.BeginningOfSentenceToken;
+ EndOfSentenceToken = options.EndOfSentenceToken;
+ UnknownToken = options.UnknownToken;
+ AddDummyPrefix = options.AddDummyPrefix;
+ EscapeWhiteSpaces = options.EscapeWhiteSpaces;
+ TreatWhitespaceAsSuffix = options.TreatWhitespaceAsSuffix;
+ ByteFallback = options.ByteFallback;
+ SpecialTokens = options.SpecialTokens;
+
+ if (SpecialTokens is not null && SpecialTokens.Count > 0)
+ {
+ InternalSpecialTokens = new Dictionary<StringSpanOrdinalKey, int>();
+ SpecialTokensReverse = new Dictionary<int, string>();
+
+ foreach (var item in SpecialTokens)
+ {
+ InternalSpecialTokens.Add(new StringSpanOrdinalKey(item.Key), item.Value);
+ SpecialTokensReverse.Add(item.Value, item.Key);
+ }
+
+ // We create this Regex object without a timeout, as we expect the match operation to complete in O(N) time complexity. Note that `specialTokens` are treated as constants after the tokenizer is created.
+ SpecialTokensRegex = new Regex(string.Join("|", SpecialTokens.Keys.Select(s => Regex.Escape(s))), RegexOptions.Compiled);
+ }
+
+ Normalizer = new SentencePieceNormalizer(
+ options.PrecompiledNormalizationData,
+ options.RemoveExtraWhiteSpaces,
+ options.AddDummyPrefix, options.EscapeWhiteSpaces,
+ options.TreatWhitespaceAsSuffix,
+ SpecialTokens);
+ }
+
  internal Regex? SpecialTokensRegex { get; }
 
  internal Dictionary<StringSpanOrdinalKey, int>? InternalSpecialTokens { get; }
@@ -91,11 +152,11 @@ internal SentencePieceBaseModel(ModelProto modelProto, bool addBos = false, bool
 
  public string UnknownToken { get; }
 
- public int BeginningOfSentenceId { get; }
+ public int BeginningOfSentenceId { get; set; }
 
- public int EndOfSentenceId { get; }
+ public int EndOfSentenceId { get; set; }
 
- public int UnknownId { get; }
+ public int UnknownId { get; set; }
 
  public SentencePieceNormalizer? Normalizer { get; }
 

src/Microsoft.ML.Tokenizers/Model/SentencePieceBpeModel.cs

@@ -41,6 +41,52 @@ internal SentencePieceBpeModel(ModelProto modelProto, bool addBos, bool addEos,
  OneByteUtf8EncodingMaxId = ByteCodeToIdOffset + 0x7F; // 0x7F is the maximum value of the one byte UTF-8 character.
  }
 
+ internal SentencePieceBpeModel(SentencePieceOptions options) : base(options)
+ {
+ if (options.PrecompiledNormalizationData is not null)
+ {
+ throw new NotSupportedException("Normalization data is not supported for SentencePieceBpeModel.");
+ }
+
+ Debug.Assert(options.Vocabulary is not null);
+
+ int id = 0;
+ foreach (var item in options.Vocabulary!)
+ {
+ _vocab.Add(new StringSpanOrdinalKey(item.Token), (id, item.Score, (byte)ModelProto.Types.SentencePiece.Types.Type.Normal));
+ _vocabReverse.Add(id++, item.Token);
+ }
+
+ if (options.ByteFallback)
+ {
+ if (!_vocab.TryGetValue("<0x00>", out (int Id, float Score, byte Type) value))
+ {
+ throw new ArgumentException("'ByteFallback' is enabled but the vocabulary must include a special token for each byte value (0-255) in the format <0xNN>, where NN represents the byte's hexadecimal value.");
+ }
+
+ ByteCodeToIdOffset = value.Id;
+ OneByteUtf8EncodingMaxId = ByteCodeToIdOffset + 0x7F; // 0x7F is the maximum value of the one byte UTF-8 character.
+ }
+
+ if (!_vocab.TryGetValue(options.UnknownToken, out (int Id, float Score, byte Type) unknownToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the unknown token '{options.UnknownToken}'.");
+ }
+ UnknownId = unknownToken.Id;
+
+ if (!_vocab.TryGetValue(options.BeginningOfSentenceToken, out (int Id, float Score, byte Type) beginOfSentenceToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the beginning of sentence token '{options.BeginningOfSentenceToken}'.");
+ }
+ BeginningOfSentenceId = beginOfSentenceToken.Id;
+
+ if (!_vocab.TryGetValue(options.EndOfSentenceToken, out (int Id, float Score, byte Type) endOfSentenceToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the end of sentence token '{options.EndOfSentenceToken}'.");
+ }
+ EndOfSentenceId = endOfSentenceToken.Id;
+ }
+
  public override IReadOnlyDictionary<string, int> Vocabulary
  {
  get

src/Microsoft.ML.Tokenizers/Model/SentencePieceOptions.cs

@@ -0,0 +1,118 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+
+namespace Microsoft.ML.Tokenizers
+{
+#pragma warning disable MSML_NoInstanceInitializers
+ /// <summary>
+ /// The type of the SentencePiece model.
+ /// </summary>
+ public enum SentencePieceModelType
+ {
+ /// <summary>
+ /// The model type is not defined.
+ /// </summary>
+ Undefined = 0,
+
+ /// <summary>
+ /// The model type is Byte Pair Encoding (Bpe) model.
+ /// </summary>
+ Bpe = 1,
+
+ /// <summary>
+ /// The model type is Unigram model.
+ /// </summary>
+ Unigram = 2,
+ }
+
+ /// <summary>
+ /// Options for the SentencePiece tokenizer.
+ /// </summary>
+ /// <remarks>
+ /// The options are used to configure the SentencePiece tokenizer. Serialization is not guaranteed for this type.
+ /// </remarks>
+ public sealed class SentencePieceOptions
+ {
+ /// <summary>
+ /// The type of the SentencePiece model.
+ /// </summary>
+ public SentencePieceModelType ModelType { get; set; }
+
+ /// <summary>
+ /// Determines whether the model uses a byte fallback strategy to encode unknown tokens as byte sequences.
+ /// </summary>
+ /// <remarks>
+ /// The vocabulary must include a special token for each byte value (0-255) in the format &lt;0xNN&gt;,
+ /// where NN represents the byte's hexadecimal value (e.g., &lt;0x41&gt; for byte value 65).
+ /// </remarks>
+ public bool ByteFallback { get; set; }
+
+ /// <summary>
+ /// Indicate emitting the prefix character e.g. U+2581 at the beginning of sentence token during the normalization and encoding.
+ /// </summary>
+ public bool AddDummyPrefix { get; set; }
+
+ /// <summary>
+ /// Indicate if the spaces should be replaced with character U+2581 during the normalization and encoding. Default value is `true`.
+ /// </summary>
+ public bool EscapeWhiteSpaces { get; set; } = true;
+
+ /// <summary>
+ /// Indicate emitting the character U+2581 at the end of the last sentence token instead beginning of sentence token during the normalization and encoding.
+ /// </summary>
+ public bool TreatWhitespaceAsSuffix { get; set; }
+
+ /// <summary>
+ /// Indicate removing extra white spaces from the original string during the normalization.
+ /// </summary>
+ public bool RemoveExtraWhiteSpaces { get; set; }
+
+ /// <summary>
+ /// Indicate emitting the beginning of sentence token during the encoding. Default value is `true`.
+ /// </summary>
+ public bool AddBeginningOfSentence { get; set; } = true;
+
+ /// <summary>
+ /// Indicate emitting the end of sentence token during the encoding.
+ /// </summary>
+ public bool AddEndOfSentence { get; set; }
+
+ /// <summary>
+ /// The beginning of sentence token. Default value is `&lt;s&gt;`.
+ /// </summary>
+ public string BeginningOfSentenceToken { get; set; } = "<s>";
+
+ /// <summary>
+ /// The end of sentence token. Default value is `&lt;/s&gt;`.
+ /// </summary>
+ public string EndOfSentenceToken { get; set; } = "</s>";
+
+ /// <summary>
+ /// The unknown token. Default value is `&lt;unk&gt;`.
+ /// </summary>
+ public string UnknownToken { get; set; } = "<unk>";
+
+ /// <summary>
+ /// The data used for string normalization.
+ /// </summary>
+ public byte[]? PrecompiledNormalizationData { get; set; }
+
+ /// <summary>
+ /// Represent the vocabulary.
+ /// The list should be sorted by token ID, with entries passed in the order that corresponds to their IDs. In other words,
+ /// the first entry in the list will be mapped to ID 0, the second entry to ID 1, the third to ID 2, and so on.
+ /// Each entry represents a token and its corresponding score.
+ /// </summary>
+ public IEnumerable<(string Token, float Score)>? Vocabulary { get; set; }
+
+ /// <summary>
+ /// The special tokens.
+ /// Special tokens remain intact during encoding and are not split into sub-tokens.
+ /// </summary>
+ public IReadOnlyDictionary<string, int>? SpecialTokens { get; set; }
+ }
+#pragma warning restore MSML_NoInstanceInitializers
+}

src/Microsoft.ML.Tokenizers/Model/SentencePieceTokenizer.cs

@@ -30,6 +30,16 @@ internal SentencePieceTokenizer(ModelProto modelProto, bool addBos, bool addEos,
  };
  }
 
+ internal SentencePieceTokenizer(SentencePieceOptions options)
+ {
+ _model = options.ModelType switch
+ {
+ SentencePieceModelType.Bpe => new SentencePieceBpeModel(options),
+ SentencePieceModelType.Unigram => new SentencePieceUnigramModel(options),
+ _ => throw new ArgumentException($"The model type '{options.ModelType}' is not supported.", nameof(options.ModelType))
+ };
+ }
+
  /// <summary>
  /// The special tokens.
  /// </summary>
@@ -457,5 +467,19 @@ public static SentencePieceTokenizer Create(
 
  return new SentencePieceTokenizer(modelProto, addBeginOfSentence, addEndOfSentence, specialTokens);
  }
+
+ /// <summary>
+ /// Creates an instance of SentencePieceTokenizer.
+ /// </summary>
+ /// <param name="options">The options to use for the sentence piece tokenizer.</param>
+ public static SentencePieceTokenizer Create(SentencePieceOptions options)
+ {
+ if (options is null)
+ {
+ throw new ArgumentNullException(nameof(options));
+ }
+
+ return new SentencePieceTokenizer(options);
+ }
  }
 }

eng/Versions.props

@@ -100,7 +100,7 @@
  <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
  <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
  <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
- <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25110.1</MicrosoftMLTestTokenizersVersion>
+ <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25126.1</MicrosoftMLTestTokenizersVersion>
  <SystemDataSqlClientVersion>4.9.0</SystemDataSqlClientVersion>
  <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
  <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>

src/Microsoft.ML.Tokenizers/Model/SentencePieceBaseModel.cs

@@ -59,6 +59,67 @@ internal SentencePieceBaseModel(ModelProto modelProto, bool addBos = false, bool
  specialTokens);
  }
 
+ internal SentencePieceBaseModel(SentencePieceOptions options)
+ {
+ if (options is null)
+ {
+ throw new ArgumentNullException(nameof(options));
+ }
+
+ if (options.Vocabulary is null)
+ {
+ throw new ArgumentNullException(nameof(options.Vocabulary));
+ }
+
+ if (options.BeginningOfSentenceToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.BeginningOfSentenceToken));
+ }
+
+ if (options.EndOfSentenceToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.EndOfSentenceToken));
+ }
+
+ if (options.UnknownToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.UnknownToken));
+ }
+
+ AddBeginningOfSentence = options.AddBeginningOfSentence;
+ AddEndOfSentence = options.AddEndOfSentence;
+ BeginningOfSentenceToken = options.BeginningOfSentenceToken;
+ EndOfSentenceToken = options.EndOfSentenceToken;
+ UnknownToken = options.UnknownToken;
+ AddDummyPrefix = options.AddDummyPrefix;
+ EscapeWhiteSpaces = options.EscapeWhiteSpaces;
+ TreatWhitespaceAsSuffix = options.TreatWhitespaceAsSuffix;
+ ByteFallback = options.ByteFallback;
+ SpecialTokens = options.SpecialTokens;
+
+ if (SpecialTokens is not null && SpecialTokens.Count > 0)
+ {
+ InternalSpecialTokens = new Dictionary<StringSpanOrdinalKey, int>();
+ SpecialTokensReverse = new Dictionary<int, string>();
+
+ foreach (var item in SpecialTokens)
+ {
+ InternalSpecialTokens.Add(new StringSpanOrdinalKey(item.Key), item.Value);
+ SpecialTokensReverse.Add(item.Value, item.Key);
+ }
+
+ // We create this Regex object without a timeout, as we expect the match operation to complete in O(N) time complexity. Note that `specialTokens` are treated as constants after the tokenizer is created.
+ SpecialTokensRegex = new Regex(string.Join("|", SpecialTokens.Keys.Select(s => Regex.Escape(s))), RegexOptions.Compiled);
+ }
+
+ Normalizer = new SentencePieceNormalizer(
+ options.PrecompiledNormalizationData,
+ options.RemoveExtraWhiteSpaces,
+ options.AddDummyPrefix, options.EscapeWhiteSpaces,
+ options.TreatWhitespaceAsSuffix,
+ SpecialTokens);
+ }
+
  internal Regex? SpecialTokensRegex { get; }
 
  internal Dictionary<StringSpanOrdinalKey, int>? InternalSpecialTokens { get; }
@@ -91,11 +152,11 @@ internal SentencePieceBaseModel(ModelProto modelProto, bool addBos = false, bool
 
  public string UnknownToken { get; }
 
- public int BeginningOfSentenceId { get; }
+ public int BeginningOfSentenceId { get; set; }
 
- public int EndOfSentenceId { get; }
+ public int EndOfSentenceId { get; set; }
 
- public int UnknownId { get; }
+ public int UnknownId { get; set; }
 
  public SentencePieceNormalizer? Normalizer { get; }
 

src/Microsoft.ML.Tokenizers/Model/SentencePieceBpeModel.cs

@@ -41,6 +41,52 @@ internal SentencePieceBpeModel(ModelProto modelProto, bool addBos, bool addEos,
  OneByteUtf8EncodingMaxId = ByteCodeToIdOffset + 0x7F; // 0x7F is the maximum value of the one byte UTF-8 character.
  }
 
+ internal SentencePieceBpeModel(SentencePieceOptions options) : base(options)
+ {
+ if (options.PrecompiledNormalizationData is not null)
+ {
+ throw new NotSupportedException("Normalization data is not supported for SentencePieceBpeModel.");
+ }
+
+ Debug.Assert(options.Vocabulary is not null);
+
+ int id = 0;
+ foreach (var item in options.Vocabulary!)
+ {
+ _vocab.Add(new StringSpanOrdinalKey(item.Token), (id, item.Score, (byte)ModelProto.Types.SentencePiece.Types.Type.Normal));
+ _vocabReverse.Add(id++, item.Token);
+ }
+
+ if (options.ByteFallback)
+ {
+ if (!_vocab.TryGetValue("<0x00>", out (int Id, float Score, byte Type) value))
+ {
+ throw new ArgumentException("'ByteFallback' is enabled but the vocabulary must include a special token for each byte value (0-255) in the format <0xNN>, where NN represents the byte's hexadecimal value.");
+ }
+
+ ByteCodeToIdOffset = value.Id;
+ OneByteUtf8EncodingMaxId = ByteCodeToIdOffset + 0x7F; // 0x7F is the maximum value of the one byte UTF-8 character.
+ }
+
+ if (!_vocab.TryGetValue(options.UnknownToken, out (int Id, float Score, byte Type) unknownToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the unknown token '{options.UnknownToken}'.");
+ }
+ UnknownId = unknownToken.Id;
+
+ if (!_vocab.TryGetValue(options.BeginningOfSentenceToken, out (int Id, float Score, byte Type) beginOfSentenceToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the beginning of sentence token '{options.BeginningOfSentenceToken}'.");
+ }
+ BeginningOfSentenceId = beginOfSentenceToken.Id;
+
+ if (!_vocab.TryGetValue(options.EndOfSentenceToken, out (int Id, float Score, byte Type) endOfSentenceToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the end of sentence token '{options.EndOfSentenceToken}'.");
+ }
+ EndOfSentenceId = endOfSentenceToken.Id;
+ }
+
  public override IReadOnlyDictionary<string, int> Vocabulary
  {
  get

src/Microsoft.ML.Tokenizers/Model/SentencePieceOptions.cs

@@ -0,0 +1,118 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+
+namespace Microsoft.ML.Tokenizers
+{
+#pragma warning disable MSML_NoInstanceInitializers
+ /// <summary>
+ /// The type of the SentencePiece model.
+ /// </summary>
+ public enum SentencePieceModelType
+ {
+ /// <summary>
+ /// The model type is not defined.
+ /// </summary>
+ Undefined = 0,
+
+ /// <summary>
+ /// The model type is Byte Pair Encoding (Bpe) model.
+ /// </summary>
+ Bpe = 1,
+
+ /// <summary>
+ /// The model type is Unigram model.
+ /// </summary>
+ Unigram = 2,
+ }
+
+ /// <summary>
+ /// Options for the SentencePiece tokenizer.
+ /// </summary>
+ /// <remarks>
+ /// The options are used to configure the SentencePiece tokenizer. Serialization is not guaranteed for this type.
+ /// </remarks>
+ public sealed class SentencePieceOptions
+ {
+ /// <summary>
+ /// The type of the SentencePiece model.
+ /// </summary>
+ public SentencePieceModelType ModelType { get; set; }
+
+ /// <summary>
+ /// Determines whether the model uses a byte fallback strategy to encode unknown tokens as byte sequences.
+ /// </summary>
+ /// <remarks>
+ /// The vocabulary must include a special token for each byte value (0-255) in the format &lt;0xNN&gt;,
+ /// where NN represents the byte's hexadecimal value (e.g., &lt;0x41&gt; for byte value 65).
+ /// </remarks>
+ public bool ByteFallback { get; set; }
+
+ /// <summary>
+ /// Indicate emitting the prefix character e.g. U+2581 at the beginning of sentence token during the normalization and encoding.
+ /// </summary>
+ public bool AddDummyPrefix { get; set; }
+
+ /// <summary>
+ /// Indicate if the spaces should be replaced with character U+2581 during the normalization and encoding. Default value is `true`.
+ /// </summary>
+ public bool EscapeWhiteSpaces { get; set; } = true;
+
+ /// <summary>
+ /// Indicate emitting the character U+2581 at the end of the last sentence token instead beginning of sentence token during the normalization and encoding.
+ /// </summary>
+ public bool TreatWhitespaceAsSuffix { get; set; }
+
+ /// <summary>
+ /// Indicate removing extra white spaces from the original string during the normalization.
+ /// </summary>
+ public bool RemoveExtraWhiteSpaces { get; set; }
+
+ /// <summary>
+ /// Indicate emitting the beginning of sentence token during the encoding. Default value is `true`.
+ /// </summary>
+ public bool AddBeginningOfSentence { get; set; } = true;
+
+ /// <summary>
+ /// Indicate emitting the end of sentence token during the encoding.
+ /// </summary>
+ public bool AddEndOfSentence { get; set; }
+
+ /// <summary>
+ /// The beginning of sentence token. Default value is `&lt;s&gt;`.
+ /// </summary>
+ public string BeginningOfSentenceToken { get; set; } = "<s>";
+
+ /// <summary>
+ /// The end of sentence token. Default value is `&lt;/s&gt;`.
+ /// </summary>
+ public string EndOfSentenceToken { get; set; } = "</s>";
+
+ /// <summary>
+ /// The unknown token. Default value is `&lt;unk&gt;`.
+ /// </summary>
+ public string UnknownToken { get; set; } = "<unk>";
+
+ /// <summary>
+ /// The data used for string normalization.
+ /// </summary>
+ public byte[]? PrecompiledNormalizationData { get; set; }
+
+ /// <summary>
+ /// Represent the vocabulary.
+ /// The list should be sorted by token ID, with entries passed in the order that corresponds to their IDs. In other words,
+ /// the first entry in the list will be mapped to ID 0, the second entry to ID 1, the third to ID 2, and so on.
+ /// Each entry represents a token and its corresponding score.
+ /// </summary>
+ public IEnumerable<(string Token, float Score)>? Vocabulary { get; set; }
+
+ /// <summary>
+ /// The special tokens.
+ /// Special tokens remain intact during encoding and are not split into sub-tokens.
+ /// </summary>
+ public IReadOnlyDictionary<string, int>? SpecialTokens { get; set; }
+ }
+#pragma warning restore MSML_NoInstanceInitializers
+}

src/Microsoft.ML.Tokenizers/Model/SentencePieceTokenizer.cs

@@ -30,6 +30,16 @@ internal SentencePieceTokenizer(ModelProto modelProto, bool addBos, bool addEos,
  };
  }
 
+ internal SentencePieceTokenizer(SentencePieceOptions options)
+ {
+ _model = options.ModelType switch
+ {
+ SentencePieceModelType.Bpe => new SentencePieceBpeModel(options),
+ SentencePieceModelType.Unigram => new SentencePieceUnigramModel(options),
+ _ => throw new ArgumentException($"The model type '{options.ModelType}' is not supported.", nameof(options.ModelType))
+ };
+ }
+
  /// <summary>
  /// The special tokens.
  /// </summary>
@@ -457,5 +467,19 @@ public static SentencePieceTokenizer Create(
 
  return new SentencePieceTokenizer(modelProto, addBeginOfSentence, addEndOfSentence, specialTokens);
  }
+
+ /// <summary>
+ /// Creates an instance of SentencePieceTokenizer.
+ /// </summary>
+ /// <param name="options">The options to use for the sentence piece tokenizer.</param>
+ public static SentencePieceTokenizer Create(SentencePieceOptions options)
+ {
+ if (options is null)
+ {
+ throw new ArgumentNullException(nameof(options));
+ }
+
+ return new SentencePieceTokenizer(options);
+ }
  }
 }

eng/Versions.props

@@ -100,7 +100,7 @@
  <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
  <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
  <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
- <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25110.1</MicrosoftMLTestTokenizersVersion>
+ <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25126.1</MicrosoftMLTestTokenizersVersion>
  <SystemDataSqlClientVersion>4.9.0</SystemDataSqlClientVersion>
  <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
  <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>

src/Microsoft.ML.Tokenizers/Model/SentencePieceBaseModel.cs

@@ -59,6 +59,67 @@ internal SentencePieceBaseModel(ModelProto modelProto, bool addBos = false, bool
  specialTokens);
  }
 
+ internal SentencePieceBaseModel(SentencePieceOptions options)
+ {
+ if (options is null)
+ {
+ throw new ArgumentNullException(nameof(options));
+ }
+
+ if (options.Vocabulary is null)
+ {
+ throw new ArgumentNullException(nameof(options.Vocabulary));
+ }
+
+ if (options.BeginningOfSentenceToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.BeginningOfSentenceToken));
+ }
+
+ if (options.EndOfSentenceToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.EndOfSentenceToken));
+ }
+
+ if (options.UnknownToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.UnknownToken));
+ }
+
+ AddBeginningOfSentence = options.AddBeginningOfSentence;
+ AddEndOfSentence = options.AddEndOfSentence;
+ BeginningOfSentenceToken = options.BeginningOfSentenceToken;
+ EndOfSentenceToken = options.EndOfSentenceToken;
+ UnknownToken = options.UnknownToken;
+ AddDummyPrefix = options.AddDummyPrefix;
+ EscapeWhiteSpaces = options.EscapeWhiteSpaces;
+ TreatWhitespaceAsSuffix = options.TreatWhitespaceAsSuffix;
+ ByteFallback = options.ByteFallback;
+ SpecialTokens = options.SpecialTokens;
+
+ if (SpecialTokens is not null && SpecialTokens.Count > 0)
+ {
+ InternalSpecialTokens = new Dictionary<StringSpanOrdinalKey, int>();
+ SpecialTokensReverse = new Dictionary<int, string>();
+
+ foreach (var item in SpecialTokens)
+ {
+ InternalSpecialTokens.Add(new StringSpanOrdinalKey(item.Key), item.Value);
+ SpecialTokensReverse.Add(item.Value, item.Key);
+ }
+
+ // We create this Regex object without a timeout, as we expect the match operation to complete in O(N) time complexity. Note that `specialTokens` are treated as constants after the tokenizer is created.
+ SpecialTokensRegex = new Regex(string.Join("|", SpecialTokens.Keys.Select(s => Regex.Escape(s))), RegexOptions.Compiled);
+ }
+
+ Normalizer = new SentencePieceNormalizer(
+ options.PrecompiledNormalizationData,
+ options.RemoveExtraWhiteSpaces,
+ options.AddDummyPrefix, options.EscapeWhiteSpaces,
+ options.TreatWhitespaceAsSuffix,
+ SpecialTokens);
+ }
+
  internal Regex? SpecialTokensRegex { get; }
 
  internal Dictionary<StringSpanOrdinalKey, int>? InternalSpecialTokens { get; }
@@ -91,11 +152,11 @@ internal SentencePieceBaseModel(ModelProto modelProto, bool addBos = false, bool
 
  public string UnknownToken { get; }
 
- public int BeginningOfSentenceId { get; }
+ public int BeginningOfSentenceId { get; set; }
 
- public int EndOfSentenceId { get; }
+ public int EndOfSentenceId { get; set; }
 
- public int UnknownId { get; }
+ public int UnknownId { get; set; }
 
  public SentencePieceNormalizer? Normalizer { get; }
 

src/Microsoft.ML.Tokenizers/Model/SentencePieceBpeModel.cs

@@ -41,6 +41,52 @@ internal SentencePieceBpeModel(ModelProto modelProto, bool addBos, bool addEos,
  OneByteUtf8EncodingMaxId = ByteCodeToIdOffset + 0x7F; // 0x7F is the maximum value of the one byte UTF-8 character.
  }
 
+ internal SentencePieceBpeModel(SentencePieceOptions options) : base(options)
+ {
+ if (options.PrecompiledNormalizationData is not null)
+ {
+ throw new NotSupportedException("Normalization data is not supported for SentencePieceBpeModel.");
+ }
+
+ Debug.Assert(options.Vocabulary is not null);
+
+ int id = 0;
+ foreach (var item in options.Vocabulary!)
+ {
+ _vocab.Add(new StringSpanOrdinalKey(item.Token), (id, item.Score, (byte)ModelProto.Types.SentencePiece.Types.Type.Normal));
+ _vocabReverse.Add(id++, item.Token);
+ }
+
+ if (options.ByteFallback)
+ {
+ if (!_vocab.TryGetValue("<0x00>", out (int Id, float Score, byte Type) value))
+ {
+ throw new ArgumentException("'ByteFallback' is enabled but the vocabulary must include a special token for each byte value (0-255) in the format <0xNN>, where NN represents the byte's hexadecimal value.");
+ }
+
+ ByteCodeToIdOffset = value.Id;
+ OneByteUtf8EncodingMaxId = ByteCodeToIdOffset + 0x7F; // 0x7F is the maximum value of the one byte UTF-8 character.
+ }
+
+ if (!_vocab.TryGetValue(options.UnknownToken, out (int Id, float Score, byte Type) unknownToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the unknown token '{options.UnknownToken}'.");
+ }
+ UnknownId = unknownToken.Id;
+
+ if (!_vocab.TryGetValue(options.BeginningOfSentenceToken, out (int Id, float Score, byte Type) beginOfSentenceToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the beginning of sentence token '{options.BeginningOfSentenceToken}'.");
+ }
+ BeginningOfSentenceId = beginOfSentenceToken.Id;
+
+ if (!_vocab.TryGetValue(options.EndOfSentenceToken, out (int Id, float Score, byte Type) endOfSentenceToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the end of sentence token '{options.EndOfSentenceToken}'.");
+ }
+ EndOfSentenceId = endOfSentenceToken.Id;
+ }
+
  public override IReadOnlyDictionary<string, int> Vocabulary
  {
  get

src/Microsoft.ML.Tokenizers/Model/SentencePieceOptions.cs

@@ -0,0 +1,118 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+
+namespace Microsoft.ML.Tokenizers
+{
+#pragma warning disable MSML_NoInstanceInitializers
+ /// <summary>
+ /// The type of the SentencePiece model.
+ /// </summary>
+ public enum SentencePieceModelType
+ {
+ /// <summary>
+ /// The model type is not defined.
+ /// </summary>
+ Undefined = 0,
+
+ /// <summary>
+ /// The model type is Byte Pair Encoding (Bpe) model.
+ /// </summary>
+ Bpe = 1,
+
+ /// <summary>
+ /// The model type is Unigram model.
+ /// </summary>
+ Unigram = 2,
+ }
+
+ /// <summary>
+ /// Options for the SentencePiece tokenizer.
+ /// </summary>
+ /// <remarks>
+ /// The options are used to configure the SentencePiece tokenizer. Serialization is not guaranteed for this type.
+ /// </remarks>
+ public sealed class SentencePieceOptions
+ {
+ /// <summary>
+ /// The type of the SentencePiece model.
+ /// </summary>
+ public SentencePieceModelType ModelType { get; set; }
+
+ /// <summary>
+ /// Determines whether the model uses a byte fallback strategy to encode unknown tokens as byte sequences.
+ /// </summary>
+ /// <remarks>
+ /// The vocabulary must include a special token for each byte value (0-255) in the format &lt;0xNN&gt;,
+ /// where NN represents the byte's hexadecimal value (e.g., &lt;0x41&gt; for byte value 65).
+ /// </remarks>
+ public bool ByteFallback { get; set; }
+
+ /// <summary>
+ /// Indicate emitting the prefix character e.g. U+2581 at the beginning of sentence token during the normalization and encoding.
+ /// </summary>
+ public bool AddDummyPrefix { get; set; }
+
+ /// <summary>
+ /// Indicate if the spaces should be replaced with character U+2581 during the normalization and encoding. Default value is `true`.
+ /// </summary>
+ public bool EscapeWhiteSpaces { get; set; } = true;
+
+ /// <summary>
+ /// Indicate emitting the character U+2581 at the end of the last sentence token instead beginning of sentence token during the normalization and encoding.
+ /// </summary>
+ public bool TreatWhitespaceAsSuffix { get; set; }
+
+ /// <summary>
+ /// Indicate removing extra white spaces from the original string during the normalization.
+ /// </summary>
+ public bool RemoveExtraWhiteSpaces { get; set; }
+
+ /// <summary>
+ /// Indicate emitting the beginning of sentence token during the encoding. Default value is `true`.
+ /// </summary>
+ public bool AddBeginningOfSentence { get; set; } = true;
+
+ /// <summary>
+ /// Indicate emitting the end of sentence token during the encoding.
+ /// </summary>
+ public bool AddEndOfSentence { get; set; }
+
+ /// <summary>
+ /// The beginning of sentence token. Default value is `&lt;s&gt;`.
+ /// </summary>
+ public string BeginningOfSentenceToken { get; set; } = "<s>";
+
+ /// <summary>
+ /// The end of sentence token. Default value is `&lt;/s&gt;`.
+ /// </summary>
+ public string EndOfSentenceToken { get; set; } = "</s>";
+
+ /// <summary>
+ /// The unknown token. Default value is `&lt;unk&gt;`.
+ /// </summary>
+ public string UnknownToken { get; set; } = "<unk>";
+
+ /// <summary>
+ /// The data used for string normalization.
+ /// </summary>
+ public byte[]? PrecompiledNormalizationData { get; set; }
+
+ /// <summary>
+ /// Represent the vocabulary.
+ /// The list should be sorted by token ID, with entries passed in the order that corresponds to their IDs. In other words,
+ /// the first entry in the list will be mapped to ID 0, the second entry to ID 1, the third to ID 2, and so on.
+ /// Each entry represents a token and its corresponding score.
+ /// </summary>
+ public IEnumerable<(string Token, float Score)>? Vocabulary { get; set; }
+
+ /// <summary>
+ /// The special tokens.
+ /// Special tokens remain intact during encoding and are not split into sub-tokens.
+ /// </summary>
+ public IReadOnlyDictionary<string, int>? SpecialTokens { get; set; }
+ }
+#pragma warning restore MSML_NoInstanceInitializers
+}

src/Microsoft.ML.Tokenizers/Model/SentencePieceTokenizer.cs

@@ -30,6 +30,16 @@ internal SentencePieceTokenizer(ModelProto modelProto, bool addBos, bool addEos,
  };
  }
 
+ internal SentencePieceTokenizer(SentencePieceOptions options)
+ {
+ _model = options.ModelType switch
+ {
+ SentencePieceModelType.Bpe => new SentencePieceBpeModel(options),
+ SentencePieceModelType.Unigram => new SentencePieceUnigramModel(options),
+ _ => throw new ArgumentException($"The model type '{options.ModelType}' is not supported.", nameof(options.ModelType))
+ };
+ }
+
  /// <summary>
  /// The special tokens.
  /// </summary>
@@ -457,5 +467,19 @@ public static SentencePieceTokenizer Create(
 
  return new SentencePieceTokenizer(modelProto, addBeginOfSentence, addEndOfSentence, specialTokens);
  }
+
+ /// <summary>
+ /// Creates an instance of SentencePieceTokenizer.
+ /// </summary>
+ /// <param name="options">The options to use for the sentence piece tokenizer.</param>
+ public static SentencePieceTokenizer Create(SentencePieceOptions options)
+ {
+ if (options is null)
+ {
+ throw new ArgumentNullException(nameof(options));
+ }
+
+ return new SentencePieceTokenizer(options);
+ }
  }
 }

eng/Versions.props

@@ -100,7 +100,7 @@
  <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
  <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
  <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
- <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25110.1</MicrosoftMLTestTokenizersVersion>
+ <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25126.1</MicrosoftMLTestTokenizersVersion>
  <SystemDataSqlClientVersion>4.9.0</SystemDataSqlClientVersion>
  <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
  <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>

src/Microsoft.ML.Tokenizers/Model/SentencePieceBaseModel.cs

@@ -59,6 +59,67 @@ internal SentencePieceBaseModel(ModelProto modelProto, bool addBos = false, bool
  specialTokens);
  }
 
+ internal SentencePieceBaseModel(SentencePieceOptions options)
+ {
+ if (options is null)
+ {
+ throw new ArgumentNullException(nameof(options));
+ }
+
+ if (options.Vocabulary is null)
+ {
+ throw new ArgumentNullException(nameof(options.Vocabulary));
+ }
+
+ if (options.BeginningOfSentenceToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.BeginningOfSentenceToken));
+ }
+
+ if (options.EndOfSentenceToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.EndOfSentenceToken));
+ }
+
+ if (options.UnknownToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.UnknownToken));
+ }
+
+ AddBeginningOfSentence = options.AddBeginningOfSentence;
+ AddEndOfSentence = options.AddEndOfSentence;
+ BeginningOfSentenceToken = options.BeginningOfSentenceToken;
+ EndOfSentenceToken = options.EndOfSentenceToken;
+ UnknownToken = options.UnknownToken;
+ AddDummyPrefix = options.AddDummyPrefix;
+ EscapeWhiteSpaces = options.EscapeWhiteSpaces;
+ TreatWhitespaceAsSuffix = options.TreatWhitespaceAsSuffix;
+ ByteFallback = options.ByteFallback;
+ SpecialTokens = options.SpecialTokens;
+
+ if (SpecialTokens is not null && SpecialTokens.Count > 0)
+ {
+ InternalSpecialTokens = new Dictionary<StringSpanOrdinalKey, int>();
+ SpecialTokensReverse = new Dictionary<int, string>();
+
+ foreach (var item in SpecialTokens)
+ {
+ InternalSpecialTokens.Add(new StringSpanOrdinalKey(item.Key), item.Value);
+ SpecialTokensReverse.Add(item.Value, item.Key);
+ }
+
+ // We create this Regex object without a timeout, as we expect the match operation to complete in O(N) time complexity. Note that `specialTokens` are treated as constants after the tokenizer is created.
+ SpecialTokensRegex = new Regex(string.Join("|", SpecialTokens.Keys.Select(s => Regex.Escape(s))), RegexOptions.Compiled);
+ }
+
+ Normalizer = new SentencePieceNormalizer(
+ options.PrecompiledNormalizationData,
+ options.RemoveExtraWhiteSpaces,
+ options.AddDummyPrefix, options.EscapeWhiteSpaces,
+ options.TreatWhitespaceAsSuffix,
+ SpecialTokens);
+ }
+
  internal Regex? SpecialTokensRegex { get; }
 
  internal Dictionary<StringSpanOrdinalKey, int>? InternalSpecialTokens { get; }
@@ -91,11 +152,11 @@ internal SentencePieceBaseModel(ModelProto modelProto, bool addBos = false, bool
 
  public string UnknownToken { get; }
 
- public int BeginningOfSentenceId { get; }
+ public int BeginningOfSentenceId { get; set; }
 
- public int EndOfSentenceId { get; }
+ public int EndOfSentenceId { get; set; }
 
- public int UnknownId { get; }
+ public int UnknownId { get; set; }
 
  public SentencePieceNormalizer? Normalizer { get; }
 

src/Microsoft.ML.Tokenizers/Model/SentencePieceBpeModel.cs

@@ -41,6 +41,52 @@ internal SentencePieceBpeModel(ModelProto modelProto, bool addBos, bool addEos,
  OneByteUtf8EncodingMaxId = ByteCodeToIdOffset + 0x7F; // 0x7F is the maximum value of the one byte UTF-8 character.
  }
 
+ internal SentencePieceBpeModel(SentencePieceOptions options) : base(options)
+ {
+ if (options.PrecompiledNormalizationData is not null)
+ {
+ throw new NotSupportedException("Normalization data is not supported for SentencePieceBpeModel.");
+ }
+
+ Debug.Assert(options.Vocabulary is not null);
+
+ int id = 0;
+ foreach (var item in options.Vocabulary!)
+ {
+ _vocab.Add(new StringSpanOrdinalKey(item.Token), (id, item.Score, (byte)ModelProto.Types.SentencePiece.Types.Type.Normal));
+ _vocabReverse.Add(id++, item.Token);
+ }
+
+ if (options.ByteFallback)
+ {
+ if (!_vocab.TryGetValue("<0x00>", out (int Id, float Score, byte Type) value))
+ {
+ throw new ArgumentException("'ByteFallback' is enabled but the vocabulary must include a special token for each byte value (0-255) in the format <0xNN>, where NN represents the byte's hexadecimal value.");
+ }
+
+ ByteCodeToIdOffset = value.Id;
+ OneByteUtf8EncodingMaxId = ByteCodeToIdOffset + 0x7F; // 0x7F is the maximum value of the one byte UTF-8 character.
+ }
+
+ if (!_vocab.TryGetValue(options.UnknownToken, out (int Id, float Score, byte Type) unknownToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the unknown token '{options.UnknownToken}'.");
+ }
+ UnknownId = unknownToken.Id;
+
+ if (!_vocab.TryGetValue(options.BeginningOfSentenceToken, out (int Id, float Score, byte Type) beginOfSentenceToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the beginning of sentence token '{options.BeginningOfSentenceToken}'.");
+ }
+ BeginningOfSentenceId = beginOfSentenceToken.Id;
+
+ if (!_vocab.TryGetValue(options.EndOfSentenceToken, out (int Id, float Score, byte Type) endOfSentenceToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the end of sentence token '{options.EndOfSentenceToken}'.");
+ }
+ EndOfSentenceId = endOfSentenceToken.Id;
+ }
+
  public override IReadOnlyDictionary<string, int> Vocabulary
  {
  get

src/Microsoft.ML.Tokenizers/Model/SentencePieceOptions.cs

@@ -0,0 +1,118 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+
+namespace Microsoft.ML.Tokenizers
+{
+#pragma warning disable MSML_NoInstanceInitializers
+ /// <summary>
+ /// The type of the SentencePiece model.
+ /// </summary>
+ public enum SentencePieceModelType
+ {
+ /// <summary>
+ /// The model type is not defined.
+ /// </summary>
+ Undefined = 0,
+
+ /// <summary>
+ /// The model type is Byte Pair Encoding (Bpe) model.
+ /// </summary>
+ Bpe = 1,
+
+ /// <summary>
+ /// The model type is Unigram model.
+ /// </summary>
+ Unigram = 2,
+ }
+
+ /// <summary>
+ /// Options for the SentencePiece tokenizer.
+ /// </summary>
+ /// <remarks>
+ /// The options are used to configure the SentencePiece tokenizer. Serialization is not guaranteed for this type.
+ /// </remarks>
+ public sealed class SentencePieceOptions
+ {
+ /// <summary>
+ /// The type of the SentencePiece model.
+ /// </summary>
+ public SentencePieceModelType ModelType { get; set; }
+
+ /// <summary>
+ /// Determines whether the model uses a byte fallback strategy to encode unknown tokens as byte sequences.
+ /// </summary>
+ /// <remarks>
+ /// The vocabulary must include a special token for each byte value (0-255) in the format &lt;0xNN&gt;,
+ /// where NN represents the byte's hexadecimal value (e.g., &lt;0x41&gt; for byte value 65).
+ /// </remarks>
+ public bool ByteFallback { get; set; }
+
+ /// <summary>
+ /// Indicate emitting the prefix character e.g. U+2581 at the beginning of sentence token during the normalization and encoding.
+ /// </summary>
+ public bool AddDummyPrefix { get; set; }
+
+ /// <summary>
+ /// Indicate if the spaces should be replaced with character U+2581 during the normalization and encoding. Default value is `true`.
+ /// </summary>
+ public bool EscapeWhiteSpaces { get; set; } = true;
+
+ /// <summary>
+ /// Indicate emitting the character U+2581 at the end of the last sentence token instead beginning of sentence token during the normalization and encoding.
+ /// </summary>
+ public bool TreatWhitespaceAsSuffix { get; set; }
+
+ /// <summary>
+ /// Indicate removing extra white spaces from the original string during the normalization.
+ /// </summary>
+ public bool RemoveExtraWhiteSpaces { get; set; }
+
+ /// <summary>
+ /// Indicate emitting the beginning of sentence token during the encoding. Default value is `true`.
+ /// </summary>
+ public bool AddBeginningOfSentence { get; set; } = true;
+
+ /// <summary>
+ /// Indicate emitting the end of sentence token during the encoding.
+ /// </summary>
+ public bool AddEndOfSentence { get; set; }
+
+ /// <summary>
+ /// The beginning of sentence token. Default value is `&lt;s&gt;`.
+ /// </summary>
+ public string BeginningOfSentenceToken { get; set; } = "<s>";
+
+ /// <summary>
+ /// The end of sentence token. Default value is `&lt;/s&gt;`.
+ /// </summary>
+ public string EndOfSentenceToken { get; set; } = "</s>";
+
+ /// <summary>
+ /// The unknown token. Default value is `&lt;unk&gt;`.
+ /// </summary>
+ public string UnknownToken { get; set; } = "<unk>";
+
+ /// <summary>
+ /// The data used for string normalization.
+ /// </summary>
+ public byte[]? PrecompiledNormalizationData { get; set; }
+
+ /// <summary>
+ /// Represent the vocabulary.
+ /// The list should be sorted by token ID, with entries passed in the order that corresponds to their IDs. In other words,
+ /// the first entry in the list will be mapped to ID 0, the second entry to ID 1, the third to ID 2, and so on.
+ /// Each entry represents a token and its corresponding score.
+ /// </summary>
+ public IEnumerable<(string Token, float Score)>? Vocabulary { get; set; }
+
+ /// <summary>
+ /// The special tokens.
+ /// Special tokens remain intact during encoding and are not split into sub-tokens.
+ /// </summary>
+ public IReadOnlyDictionary<string, int>? SpecialTokens { get; set; }
+ }
+#pragma warning restore MSML_NoInstanceInitializers
+}

src/Microsoft.ML.Tokenizers/Model/SentencePieceTokenizer.cs

@@ -30,6 +30,16 @@ internal SentencePieceTokenizer(ModelProto modelProto, bool addBos, bool addEos,
  };
  }
 
+ internal SentencePieceTokenizer(SentencePieceOptions options)
+ {
+ _model = options.ModelType switch
+ {
+ SentencePieceModelType.Bpe => new SentencePieceBpeModel(options),
+ SentencePieceModelType.Unigram => new SentencePieceUnigramModel(options),
+ _ => throw new ArgumentException($"The model type '{options.ModelType}' is not supported.", nameof(options.ModelType))
+ };
+ }
+
  /// <summary>
  /// The special tokens.
  /// </summary>
@@ -457,5 +467,19 @@ public static SentencePieceTokenizer Create(
 
  return new SentencePieceTokenizer(modelProto, addBeginOfSentence, addEndOfSentence, specialTokens);
  }
+
+ /// <summary>
+ /// Creates an instance of SentencePieceTokenizer.
+ /// </summary>
+ /// <param name="options">The options to use for the sentence piece tokenizer.</param>
+ public static SentencePieceTokenizer Create(SentencePieceOptions options)
+ {
+ if (options is null)
+ {
+ throw new ArgumentNullException(nameof(options));
+ }
+
+ return new SentencePieceTokenizer(options);
+ }
  }
 }

eng/Versions.props

@@ -100,7 +100,7 @@
  <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
  <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
  <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
- <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25110.1</MicrosoftMLTestTokenizersVersion>
+ <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25126.1</MicrosoftMLTestTokenizersVersion>
  <SystemDataSqlClientVersion>4.9.0</SystemDataSqlClientVersion>
  <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
  <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>

src/Microsoft.ML.Tokenizers/Model/SentencePieceBaseModel.cs

@@ -59,6 +59,67 @@ internal SentencePieceBaseModel(ModelProto modelProto, bool addBos = false, bool
  specialTokens);
  }
 
+ internal SentencePieceBaseModel(SentencePieceOptions options)
+ {
+ if (options is null)
+ {
+ throw new ArgumentNullException(nameof(options));
+ }
+
+ if (options.Vocabulary is null)
+ {
+ throw new ArgumentNullException(nameof(options.Vocabulary));
+ }
+
+ if (options.BeginningOfSentenceToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.BeginningOfSentenceToken));
+ }
+
+ if (options.EndOfSentenceToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.EndOfSentenceToken));
+ }
+
+ if (options.UnknownToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.UnknownToken));
+ }
+
+ AddBeginningOfSentence = options.AddBeginningOfSentence;
+ AddEndOfSentence = options.AddEndOfSentence;
+ BeginningOfSentenceToken = options.BeginningOfSentenceToken;
+ EndOfSentenceToken = options.EndOfSentenceToken;
+ UnknownToken = options.UnknownToken;
+ AddDummyPrefix = options.AddDummyPrefix;
+ EscapeWhiteSpaces = options.EscapeWhiteSpaces;
+ TreatWhitespaceAsSuffix = options.TreatWhitespaceAsSuffix;
+ ByteFallback = options.ByteFallback;
+ SpecialTokens = options.SpecialTokens;
+
+ if (SpecialTokens is not null && SpecialTokens.Count > 0)
+ {
+ InternalSpecialTokens = new Dictionary<StringSpanOrdinalKey, int>();
+ SpecialTokensReverse = new Dictionary<int, string>();
+
+ foreach (var item in SpecialTokens)
+ {
+ InternalSpecialTokens.Add(new StringSpanOrdinalKey(item.Key), item.Value);
+ SpecialTokensReverse.Add(item.Value, item.Key);
+ }
+
+ // We create this Regex object without a timeout, as we expect the match operation to complete in O(N) time complexity. Note that `specialTokens` are treated as constants after the tokenizer is created.
+ SpecialTokensRegex = new Regex(string.Join("|", SpecialTokens.Keys.Select(s => Regex.Escape(s))), RegexOptions.Compiled);
+ }
+
+ Normalizer = new SentencePieceNormalizer(
+ options.PrecompiledNormalizationData,
+ options.RemoveExtraWhiteSpaces,
+ options.AddDummyPrefix, options.EscapeWhiteSpaces,
+ options.TreatWhitespaceAsSuffix,
+ SpecialTokens);
+ }
+
  internal Regex? SpecialTokensRegex { get; }
 
  internal Dictionary<StringSpanOrdinalKey, int>? InternalSpecialTokens { get; }
@@ -91,11 +152,11 @@ internal SentencePieceBaseModel(ModelProto modelProto, bool addBos = false, bool
 
  public string UnknownToken { get; }
 
- public int BeginningOfSentenceId { get; }
+ public int BeginningOfSentenceId { get; set; }
 
- public int EndOfSentenceId { get; }
+ public int EndOfSentenceId { get; set; }
 
- public int UnknownId { get; }
+ public int UnknownId { get; set; }
 
  public SentencePieceNormalizer? Normalizer { get; }
 

src/Microsoft.ML.Tokenizers/Model/SentencePieceBpeModel.cs

@@ -41,6 +41,52 @@ internal SentencePieceBpeModel(ModelProto modelProto, bool addBos, bool addEos,
  OneByteUtf8EncodingMaxId = ByteCodeToIdOffset + 0x7F; // 0x7F is the maximum value of the one byte UTF-8 character.
  }
 
+ internal SentencePieceBpeModel(SentencePieceOptions options) : base(options)
+ {
+ if (options.PrecompiledNormalizationData is not null)
+ {
+ throw new NotSupportedException("Normalization data is not supported for SentencePieceBpeModel.");
+ }
+
+ Debug.Assert(options.Vocabulary is not null);
+
+ int id = 0;
+ foreach (var item in options.Vocabulary!)
+ {
+ _vocab.Add(new StringSpanOrdinalKey(item.Token), (id, item.Score, (byte)ModelProto.Types.SentencePiece.Types.Type.Normal));
+ _vocabReverse.Add(id++, item.Token);
+ }
+
+ if (options.ByteFallback)
+ {
+ if (!_vocab.TryGetValue("<0x00>", out (int Id, float Score, byte Type) value))
+ {
+ throw new ArgumentException("'ByteFallback' is enabled but the vocabulary must include a special token for each byte value (0-255) in the format <0xNN>, where NN represents the byte's hexadecimal value.");
+ }
+
+ ByteCodeToIdOffset = value.Id;
+ OneByteUtf8EncodingMaxId = ByteCodeToIdOffset + 0x7F; // 0x7F is the maximum value of the one byte UTF-8 character.
+ }
+
+ if (!_vocab.TryGetValue(options.UnknownToken, out (int Id, float Score, byte Type) unknownToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the unknown token '{options.UnknownToken}'.");
+ }
+ UnknownId = unknownToken.Id;
+
+ if (!_vocab.TryGetValue(options.BeginningOfSentenceToken, out (int Id, float Score, byte Type) beginOfSentenceToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the beginning of sentence token '{options.BeginningOfSentenceToken}'.");
+ }
+ BeginningOfSentenceId = beginOfSentenceToken.Id;
+
+ if (!_vocab.TryGetValue(options.EndOfSentenceToken, out (int Id, float Score, byte Type) endOfSentenceToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the end of sentence token '{options.EndOfSentenceToken}'.");
+ }
+ EndOfSentenceId = endOfSentenceToken.Id;
+ }
+
  public override IReadOnlyDictionary<string, int> Vocabulary
  {
  get

src/Microsoft.ML.Tokenizers/Model/SentencePieceOptions.cs

@@ -0,0 +1,118 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+
+namespace Microsoft.ML.Tokenizers
+{
+#pragma warning disable MSML_NoInstanceInitializers
+ /// <summary>
+ /// The type of the SentencePiece model.
+ /// </summary>
+ public enum SentencePieceModelType
+ {
+ /// <summary>
+ /// The model type is not defined.
+ /// </summary>
+ Undefined = 0,
+
+ /// <summary>
+ /// The model type is Byte Pair Encoding (Bpe) model.
+ /// </summary>
+ Bpe = 1,
+
+ /// <summary>
+ /// The model type is Unigram model.
+ /// </summary>
+ Unigram = 2,
+ }
+
+ /// <summary>
+ /// Options for the SentencePiece tokenizer.
+ /// </summary>
+ /// <remarks>
+ /// The options are used to configure the SentencePiece tokenizer. Serialization is not guaranteed for this type.
+ /// </remarks>
+ public sealed class SentencePieceOptions
+ {
+ /// <summary>
+ /// The type of the SentencePiece model.
+ /// </summary>
+ public SentencePieceModelType ModelType { get; set; }
+
+ /// <summary>
+ /// Determines whether the model uses a byte fallback strategy to encode unknown tokens as byte sequences.
+ /// </summary>
+ /// <remarks>
+ /// The vocabulary must include a special token for each byte value (0-255) in the format &lt;0xNN&gt;,
+ /// where NN represents the byte's hexadecimal value (e.g., &lt;0x41&gt; for byte value 65).
+ /// </remarks>
+ public bool ByteFallback { get; set; }
+
+ /// <summary>
+ /// Indicate emitting the prefix character e.g. U+2581 at the beginning of sentence token during the normalization and encoding.
+ /// </summary>
+ public bool AddDummyPrefix { get; set; }
+
+ /// <summary>
+ /// Indicate if the spaces should be replaced with character U+2581 during the normalization and encoding. Default value is `true`.
+ /// </summary>
+ public bool EscapeWhiteSpaces { get; set; } = true;
+
+ /// <summary>
+ /// Indicate emitting the character U+2581 at the end of the last sentence token instead beginning of sentence token during the normalization and encoding.
+ /// </summary>
+ public bool TreatWhitespaceAsSuffix { get; set; }
+
+ /// <summary>
+ /// Indicate removing extra white spaces from the original string during the normalization.
+ /// </summary>
+ public bool RemoveExtraWhiteSpaces { get; set; }
+
+ /// <summary>
+ /// Indicate emitting the beginning of sentence token during the encoding. Default value is `true`.
+ /// </summary>
+ public bool AddBeginningOfSentence { get; set; } = true;
+
+ /// <summary>
+ /// Indicate emitting the end of sentence token during the encoding.
+ /// </summary>
+ public bool AddEndOfSentence { get; set; }
+
+ /// <summary>
+ /// The beginning of sentence token. Default value is `&lt;s&gt;`.
+ /// </summary>
+ public string BeginningOfSentenceToken { get; set; } = "<s>";
+
+ /// <summary>
+ /// The end of sentence token. Default value is `&lt;/s&gt;`.
+ /// </summary>
+ public string EndOfSentenceToken { get; set; } = "</s>";
+
+ /// <summary>
+ /// The unknown token. Default value is `&lt;unk&gt;`.
+ /// </summary>
+ public string UnknownToken { get; set; } = "<unk>";
+
+ /// <summary>
+ /// The data used for string normalization.
+ /// </summary>
+ public byte[]? PrecompiledNormalizationData { get; set; }
+
+ /// <summary>
+ /// Represent the vocabulary.
+ /// The list should be sorted by token ID, with entries passed in the order that corresponds to their IDs. In other words,
+ /// the first entry in the list will be mapped to ID 0, the second entry to ID 1, the third to ID 2, and so on.
+ /// Each entry represents a token and its corresponding score.
+ /// </summary>
+ public IEnumerable<(string Token, float Score)>? Vocabulary { get; set; }
+
+ /// <summary>
+ /// The special tokens.
+ /// Special tokens remain intact during encoding and are not split into sub-tokens.
+ /// </summary>
+ public IReadOnlyDictionary<string, int>? SpecialTokens { get; set; }
+ }
+#pragma warning restore MSML_NoInstanceInitializers
+}

src/Microsoft.ML.Tokenizers/Model/SentencePieceTokenizer.cs

@@ -30,6 +30,16 @@ internal SentencePieceTokenizer(ModelProto modelProto, bool addBos, bool addEos,
  };
  }
 
+ internal SentencePieceTokenizer(SentencePieceOptions options)
+ {
+ _model = options.ModelType switch
+ {
+ SentencePieceModelType.Bpe => new SentencePieceBpeModel(options),
+ SentencePieceModelType.Unigram => new SentencePieceUnigramModel(options),
+ _ => throw new ArgumentException($"The model type '{options.ModelType}' is not supported.", nameof(options.ModelType))
+ };
+ }
+
  /// <summary>
  /// The special tokens.
  /// </summary>
@@ -457,5 +467,19 @@ public static SentencePieceTokenizer Create(
 
  return new SentencePieceTokenizer(modelProto, addBeginOfSentence, addEndOfSentence, specialTokens);
  }
+
+ /// <summary>
+ /// Creates an instance of SentencePieceTokenizer.
+ /// </summary>
+ /// <param name="options">The options to use for the sentence piece tokenizer.</param>
+ public static SentencePieceTokenizer Create(SentencePieceOptions options)
+ {
+ if (options is null)
+ {
+ throw new ArgumentNullException(nameof(options));
+ }
+
+ return new SentencePieceTokenizer(options);
+ }
  }
 }

eng/Versions.props

@@ -100,7 +100,7 @@
  <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
  <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
  <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
- <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25110.1</MicrosoftMLTestTokenizersVersion>
+ <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25126.1</MicrosoftMLTestTokenizersVersion>
  <SystemDataSqlClientVersion>4.9.0</SystemDataSqlClientVersion>
  <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
  <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>

src/Microsoft.ML.Tokenizers/Model/SentencePieceBaseModel.cs

@@ -59,6 +59,67 @@ internal SentencePieceBaseModel(ModelProto modelProto, bool addBos = false, bool
  specialTokens);
  }
 
+ internal SentencePieceBaseModel(SentencePieceOptions options)
+ {
+ if (options is null)
+ {
+ throw new ArgumentNullException(nameof(options));
+ }
+
+ if (options.Vocabulary is null)
+ {
+ throw new ArgumentNullException(nameof(options.Vocabulary));
+ }
+
+ if (options.BeginningOfSentenceToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.BeginningOfSentenceToken));
+ }
+
+ if (options.EndOfSentenceToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.EndOfSentenceToken));
+ }
+
+ if (options.UnknownToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.UnknownToken));
+ }
+
+ AddBeginningOfSentence = options.AddBeginningOfSentence;
+ AddEndOfSentence = options.AddEndOfSentence;
+ BeginningOfSentenceToken = options.BeginningOfSentenceToken;
+ EndOfSentenceToken = options.EndOfSentenceToken;
+ UnknownToken = options.UnknownToken;
+ AddDummyPrefix = options.AddDummyPrefix;
+ EscapeWhiteSpaces = options.EscapeWhiteSpaces;
+ TreatWhitespaceAsSuffix = options.TreatWhitespaceAsSuffix;
+ ByteFallback = options.ByteFallback;
+ SpecialTokens = options.SpecialTokens;
+
+ if (SpecialTokens is not null && SpecialTokens.Count > 0)
+ {
+ InternalSpecialTokens = new Dictionary<StringSpanOrdinalKey, int>();
+ SpecialTokensReverse = new Dictionary<int, string>();
+
+ foreach (var item in SpecialTokens)
+ {
+ InternalSpecialTokens.Add(new StringSpanOrdinalKey(item.Key), item.Value);
+ SpecialTokensReverse.Add(item.Value, item.Key);
+ }
+
+ // We create this Regex object without a timeout, as we expect the match operation to complete in O(N) time complexity. Note that `specialTokens` are treated as constants after the tokenizer is created.
+ SpecialTokensRegex = new Regex(string.Join("|", SpecialTokens.Keys.Select(s => Regex.Escape(s))), RegexOptions.Compiled);
+ }
+
+ Normalizer = new SentencePieceNormalizer(
+ options.PrecompiledNormalizationData,
+ options.RemoveExtraWhiteSpaces,
+ options.AddDummyPrefix, options.EscapeWhiteSpaces,
+ options.TreatWhitespaceAsSuffix,
+ SpecialTokens);
+ }
+
  internal Regex? SpecialTokensRegex { get; }
 
  internal Dictionary<StringSpanOrdinalKey, int>? InternalSpecialTokens { get; }
@@ -91,11 +152,11 @@ internal SentencePieceBaseModel(ModelProto modelProto, bool addBos = false, bool
 
  public string UnknownToken { get; }
 
- public int BeginningOfSentenceId { get; }
+ public int BeginningOfSentenceId { get; set; }
 
- public int EndOfSentenceId { get; }
+ public int EndOfSentenceId { get; set; }
 
- public int UnknownId { get; }
+ public int UnknownId { get; set; }
 
  public SentencePieceNormalizer? Normalizer { get; }
 

src/Microsoft.ML.Tokenizers/Model/SentencePieceBpeModel.cs

@@ -41,6 +41,52 @@ internal SentencePieceBpeModel(ModelProto modelProto, bool addBos, bool addEos,
  OneByteUtf8EncodingMaxId = ByteCodeToIdOffset + 0x7F; // 0x7F is the maximum value of the one byte UTF-8 character.
  }
 
+ internal SentencePieceBpeModel(SentencePieceOptions options) : base(options)
+ {
+ if (options.PrecompiledNormalizationData is not null)
+ {
+ throw new NotSupportedException("Normalization data is not supported for SentencePieceBpeModel.");
+ }
+
+ Debug.Assert(options.Vocabulary is not null);
+
+ int id = 0;
+ foreach (var item in options.Vocabulary!)
+ {
+ _vocab.Add(new StringSpanOrdinalKey(item.Token), (id, item.Score, (byte)ModelProto.Types.SentencePiece.Types.Type.Normal));
+ _vocabReverse.Add(id++, item.Token);
+ }
+
+ if (options.ByteFallback)
+ {
+ if (!_vocab.TryGetValue("<0x00>", out (int Id, float Score, byte Type) value))
+ {
+ throw new ArgumentException("'ByteFallback' is enabled but the vocabulary must include a special token for each byte value (0-255) in the format <0xNN>, where NN represents the byte's hexadecimal value.");
+ }
+
+ ByteCodeToIdOffset = value.Id;
+ OneByteUtf8EncodingMaxId = ByteCodeToIdOffset + 0x7F; // 0x7F is the maximum value of the one byte UTF-8 character.
+ }
+
+ if (!_vocab.TryGetValue(options.UnknownToken, out (int Id, float Score, byte Type) unknownToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the unknown token '{options.UnknownToken}'.");
+ }
+ UnknownId = unknownToken.Id;
+
+ if (!_vocab.TryGetValue(options.BeginningOfSentenceToken, out (int Id, float Score, byte Type) beginOfSentenceToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the beginning of sentence token '{options.BeginningOfSentenceToken}'.");
+ }
+ BeginningOfSentenceId = beginOfSentenceToken.Id;
+
+ if (!_vocab.TryGetValue(options.EndOfSentenceToken, out (int Id, float Score, byte Type) endOfSentenceToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the end of sentence token '{options.EndOfSentenceToken}'.");
+ }
+ EndOfSentenceId = endOfSentenceToken.Id;
+ }
+
  public override IReadOnlyDictionary<string, int> Vocabulary
  {
  get

src/Microsoft.ML.Tokenizers/Model/SentencePieceOptions.cs

@@ -0,0 +1,118 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+
+namespace Microsoft.ML.Tokenizers
+{
+#pragma warning disable MSML_NoInstanceInitializers
+ /// <summary>
+ /// The type of the SentencePiece model.
+ /// </summary>
+ public enum SentencePieceModelType
+ {
+ /// <summary>
+ /// The model type is not defined.
+ /// </summary>
+ Undefined = 0,
+
+ /// <summary>
+ /// The model type is Byte Pair Encoding (Bpe) model.
+ /// </summary>
+ Bpe = 1,
+
+ /// <summary>
+ /// The model type is Unigram model.
+ /// </summary>
+ Unigram = 2,
+ }
+
+ /// <summary>
+ /// Options for the SentencePiece tokenizer.
+ /// </summary>
+ /// <remarks>
+ /// The options are used to configure the SentencePiece tokenizer. Serialization is not guaranteed for this type.
+ /// </remarks>
+ public sealed class SentencePieceOptions
+ {
+ /// <summary>
+ /// The type of the SentencePiece model.
+ /// </summary>
+ public SentencePieceModelType ModelType { get; set; }
+
+ /// <summary>
+ /// Determines whether the model uses a byte fallback strategy to encode unknown tokens as byte sequences.
+ /// </summary>
+ /// <remarks>
+ /// The vocabulary must include a special token for each byte value (0-255) in the format &lt;0xNN&gt;,
+ /// where NN represents the byte's hexadecimal value (e.g., &lt;0x41&gt; for byte value 65).
+ /// </remarks>
+ public bool ByteFallback { get; set; }
+
+ /// <summary>
+ /// Indicate emitting the prefix character e.g. U+2581 at the beginning of sentence token during the normalization and encoding.
+ /// </summary>
+ public bool AddDummyPrefix { get; set; }
+
+ /// <summary>
+ /// Indicate if the spaces should be replaced with character U+2581 during the normalization and encoding. Default value is `true`.
+ /// </summary>
+ public bool EscapeWhiteSpaces { get; set; } = true;
+
+ /// <summary>
+ /// Indicate emitting the character U+2581 at the end of the last sentence token instead beginning of sentence token during the normalization and encoding.
+ /// </summary>
+ public bool TreatWhitespaceAsSuffix { get; set; }
+
+ /// <summary>
+ /// Indicate removing extra white spaces from the original string during the normalization.
+ /// </summary>
+ public bool RemoveExtraWhiteSpaces { get; set; }
+
+ /// <summary>
+ /// Indicate emitting the beginning of sentence token during the encoding. Default value is `true`.
+ /// </summary>
+ public bool AddBeginningOfSentence { get; set; } = true;
+
+ /// <summary>
+ /// Indicate emitting the end of sentence token during the encoding.
+ /// </summary>
+ public bool AddEndOfSentence { get; set; }
+
+ /// <summary>
+ /// The beginning of sentence token. Default value is `&lt;s&gt;`.
+ /// </summary>
+ public string BeginningOfSentenceToken { get; set; } = "<s>";
+
+ /// <summary>
+ /// The end of sentence token. Default value is `&lt;/s&gt;`.
+ /// </summary>
+ public string EndOfSentenceToken { get; set; } = "</s>";
+
+ /// <summary>
+ /// The unknown token. Default value is `&lt;unk&gt;`.
+ /// </summary>
+ public string UnknownToken { get; set; } = "<unk>";
+
+ /// <summary>
+ /// The data used for string normalization.
+ /// </summary>
+ public byte[]? PrecompiledNormalizationData { get; set; }
+
+ /// <summary>
+ /// Represent the vocabulary.
+ /// The list should be sorted by token ID, with entries passed in the order that corresponds to their IDs. In other words,
+ /// the first entry in the list will be mapped to ID 0, the second entry to ID 1, the third to ID 2, and so on.
+ /// Each entry represents a token and its corresponding score.
+ /// </summary>
+ public IEnumerable<(string Token, float Score)>? Vocabulary { get; set; }
+
+ /// <summary>
+ /// The special tokens.
+ /// Special tokens remain intact during encoding and are not split into sub-tokens.
+ /// </summary>
+ public IReadOnlyDictionary<string, int>? SpecialTokens { get; set; }
+ }
+#pragma warning restore MSML_NoInstanceInitializers
+}

src/Microsoft.ML.Tokenizers/Model/SentencePieceTokenizer.cs

@@ -30,6 +30,16 @@ internal SentencePieceTokenizer(ModelProto modelProto, bool addBos, bool addEos,
  };
  }
 
+ internal SentencePieceTokenizer(SentencePieceOptions options)
+ {
+ _model = options.ModelType switch
+ {
+ SentencePieceModelType.Bpe => new SentencePieceBpeModel(options),
+ SentencePieceModelType.Unigram => new SentencePieceUnigramModel(options),
+ _ => throw new ArgumentException($"The model type '{options.ModelType}' is not supported.", nameof(options.ModelType))
+ };
+ }
+
  /// <summary>
  /// The special tokens.
  /// </summary>
@@ -457,5 +467,19 @@ public static SentencePieceTokenizer Create(
 
  return new SentencePieceTokenizer(modelProto, addBeginOfSentence, addEndOfSentence, specialTokens);
  }
+
+ /// <summary>
+ /// Creates an instance of SentencePieceTokenizer.
+ /// </summary>
+ /// <param name="options">The options to use for the sentence piece tokenizer.</param>
+ public static SentencePieceTokenizer Create(SentencePieceOptions options)
+ {
+ if (options is null)
+ {
+ throw new ArgumentNullException(nameof(options));
+ }
+
+ return new SentencePieceTokenizer(options);
+ }
  }
 }

eng/Versions.props

@@ -100,7 +100,7 @@
  <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
  <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
  <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
- <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25110.1</MicrosoftMLTestTokenizersVersion>
+ <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25126.1</MicrosoftMLTestTokenizersVersion>
  <SystemDataSqlClientVersion>4.9.0</SystemDataSqlClientVersion>
  <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
  <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>

src/Microsoft.ML.Tokenizers/Model/SentencePieceBaseModel.cs

@@ -59,6 +59,67 @@ internal SentencePieceBaseModel(ModelProto modelProto, bool addBos = false, bool
  specialTokens);
  }
 
+ internal SentencePieceBaseModel(SentencePieceOptions options)
+ {
+ if (options is null)
+ {
+ throw new ArgumentNullException(nameof(options));
+ }
+
+ if (options.Vocabulary is null)
+ {
+ throw new ArgumentNullException(nameof(options.Vocabulary));
+ }
+
+ if (options.BeginningOfSentenceToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.BeginningOfSentenceToken));
+ }
+
+ if (options.EndOfSentenceToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.EndOfSentenceToken));
+ }
+
+ if (options.UnknownToken is null)
+ {
+ throw new ArgumentNullException(nameof(options.UnknownToken));
+ }
+
+ AddBeginningOfSentence = options.AddBeginningOfSentence;
+ AddEndOfSentence = options.AddEndOfSentence;
+ BeginningOfSentenceToken = options.BeginningOfSentenceToken;
+ EndOfSentenceToken = options.EndOfSentenceToken;
+ UnknownToken = options.UnknownToken;
+ AddDummyPrefix = options.AddDummyPrefix;
+ EscapeWhiteSpaces = options.EscapeWhiteSpaces;
+ TreatWhitespaceAsSuffix = options.TreatWhitespaceAsSuffix;
+ ByteFallback = options.ByteFallback;
+ SpecialTokens = options.SpecialTokens;
+
+ if (SpecialTokens is not null && SpecialTokens.Count > 0)
+ {
+ InternalSpecialTokens = new Dictionary<StringSpanOrdinalKey, int>();
+ SpecialTokensReverse = new Dictionary<int, string>();
+
+ foreach (var item in SpecialTokens)
+ {
+ InternalSpecialTokens.Add(new StringSpanOrdinalKey(item.Key), item.Value);
+ SpecialTokensReverse.Add(item.Value, item.Key);
+ }
+
+ // We create this Regex object without a timeout, as we expect the match operation to complete in O(N) time complexity. Note that `specialTokens` are treated as constants after the tokenizer is created.
+ SpecialTokensRegex = new Regex(string.Join("|", SpecialTokens.Keys.Select(s => Regex.Escape(s))), RegexOptions.Compiled);
+ }
+
+ Normalizer = new SentencePieceNormalizer(
+ options.PrecompiledNormalizationData,
+ options.RemoveExtraWhiteSpaces,
+ options.AddDummyPrefix, options.EscapeWhiteSpaces,
+ options.TreatWhitespaceAsSuffix,
+ SpecialTokens);
+ }
+
  internal Regex? SpecialTokensRegex { get; }
 
  internal Dictionary<StringSpanOrdinalKey, int>? InternalSpecialTokens { get; }
@@ -91,11 +152,11 @@ internal SentencePieceBaseModel(ModelProto modelProto, bool addBos = false, bool
 
  public string UnknownToken { get; }
 
- public int BeginningOfSentenceId { get; }
+ public int BeginningOfSentenceId { get; set; }
 
- public int EndOfSentenceId { get; }
+ public int EndOfSentenceId { get; set; }
 
- public int UnknownId { get; }
+ public int UnknownId { get; set; }
 
  public SentencePieceNormalizer? Normalizer { get; }
 

src/Microsoft.ML.Tokenizers/Model/SentencePieceBpeModel.cs

@@ -41,6 +41,52 @@ internal SentencePieceBpeModel(ModelProto modelProto, bool addBos, bool addEos,
  OneByteUtf8EncodingMaxId = ByteCodeToIdOffset + 0x7F; // 0x7F is the maximum value of the one byte UTF-8 character.
  }
 
+ internal SentencePieceBpeModel(SentencePieceOptions options) : base(options)
+ {
+ if (options.PrecompiledNormalizationData is not null)
+ {
+ throw new NotSupportedException("Normalization data is not supported for SentencePieceBpeModel.");
+ }
+
+ Debug.Assert(options.Vocabulary is not null);
+
+ int id = 0;
+ foreach (var item in options.Vocabulary!)
+ {
+ _vocab.Add(new StringSpanOrdinalKey(item.Token), (id, item.Score, (byte)ModelProto.Types.SentencePiece.Types.Type.Normal));
+ _vocabReverse.Add(id++, item.Token);
+ }
+
+ if (options.ByteFallback)
+ {
+ if (!_vocab.TryGetValue("<0x00>", out (int Id, float Score, byte Type) value))
+ {
+ throw new ArgumentException("'ByteFallback' is enabled but the vocabulary must include a special token for each byte value (0-255) in the format <0xNN>, where NN represents the byte's hexadecimal value.");
+ }
+
+ ByteCodeToIdOffset = value.Id;
+ OneByteUtf8EncodingMaxId = ByteCodeToIdOffset + 0x7F; // 0x7F is the maximum value of the one byte UTF-8 character.
+ }
+
+ if (!_vocab.TryGetValue(options.UnknownToken, out (int Id, float Score, byte Type) unknownToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the unknown token '{options.UnknownToken}'.");
+ }
+ UnknownId = unknownToken.Id;
+
+ if (!_vocab.TryGetValue(options.BeginningOfSentenceToken, out (int Id, float Score, byte Type) beginOfSentenceToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the beginning of sentence token '{options.BeginningOfSentenceToken}'.");
+ }
+ BeginningOfSentenceId = beginOfSentenceToken.Id;
+
+ if (!_vocab.TryGetValue(options.EndOfSentenceToken, out (int Id, float Score, byte Type) endOfSentenceToken))
+ {
+ throw new ArgumentException($"The vocabulary must include the end of sentence token '{options.EndOfSentenceToken}'.");
+ }
+ EndOfSentenceId = endOfSentenceToken.Id;
+ }
+
  public override IReadOnlyDictionary<string, int> Vocabulary
  {
  get

src/Microsoft.ML.Tokenizers/Model/SentencePieceOptions.cs

@@ -0,0 +1,118 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+
+namespace Microsoft.ML.Tokenizers
+{
+#pragma warning disable MSML_NoInstanceInitializers
+ /// <summary>
+ /// The type of the SentencePiece model.
+ /// </summary>
+ public enum SentencePieceModelType
+ {
+ /// <summary>
+ /// The model type is not defined.
+ /// </summary>
+ Undefined = 0,
+
+ /// <summary>
+ /// The model type is Byte Pair Encoding (Bpe) model.
+ /// </summary>
+ Bpe = 1,
+
+ /// <summary>
+ /// The model type is Unigram model.
+ /// </summary>
+ Unigram = 2,
+ }
+
+ /// <summary>
+ /// Options for the SentencePiece tokenizer.
+ /// </summary>
+ /// <remarks>
+ /// The options are used to configure the SentencePiece tokenizer. Serialization is not guaranteed for this type.
+ /// </remarks>
+ public sealed class SentencePieceOptions
+ {
+ /// <summary>
+ /// The type of the SentencePiece model.
+ /// </summary>
+ public SentencePieceModelType ModelType { get; set; }
+
+ /// <summary>
+ /// Determines whether the model uses a byte fallback strategy to encode unknown tokens as byte sequences.
+ /// </summary>
+ /// <remarks>
+ /// The vocabulary must include a special token for each byte value (0-255) in the format &lt;0xNN&gt;,
+ /// where NN represents the byte's hexadecimal value (e.g., &lt;0x41&gt; for byte value 65).
+ /// </remarks>
+ public bool ByteFallback { get; set; }
+
+ /// <summary>
+ /// Indicate emitting the prefix character e.g. U+2581 at the beginning of sentence token during the normalization and encoding.
+ /// </summary>
+ public bool AddDummyPrefix { get; set; }
+
+ /// <summary>
+ /// Indicate if the spaces should be replaced with character U+2581 during the normalization and encoding. Default value is `true`.
+ /// </summary>
+ public bool EscapeWhiteSpaces { get; set; } = true;
+
+ /// <summary>
+ /// Indicate emitting the character U+2581 at the end of the last sentence token instead beginning of sentence token during the normalization and encoding.
+ /// </summary>
+ public bool TreatWhitespaceAsSuffix { get; set; }
+
+ /// <summary>
+ /// Indicate removing extra white spaces from the original string during the normalization.
+ /// </summary>
+ public bool RemoveExtraWhiteSpaces { get; set; }
+
+ /// <summary>
+ /// Indicate emitting the beginning of sentence token during the encoding. Default value is `true`.
+ /// </summary>
+ public bool AddBeginningOfSentence { get; set; } = true;
+
+ /// <summary>
+ /// Indicate emitting the end of sentence token during the encoding.
+ /// </summary>
+ public bool AddEndOfSentence { get; set; }
+
+ /// <summary>
+ /// The beginning of sentence token. Default value is `&lt;s&gt;`.
+ /// </summary>
+ public string BeginningOfSentenceToken { get; set; } = "<s>";
+
+ /// <summary>
+ /// The end of sentence token. Default value is `&lt;/s&gt;`.
+ /// </summary>
+ public string EndOfSentenceToken { get; set; } = "</s>";
+
+ /// <summary>
+ /// The unknown token. Default value is `&lt;unk&gt;`.
+ /// </summary>
+ public string UnknownToken { get; set; } = "<unk>";
+
+ /// <summary>
+ /// The data used for string normalization.
+ /// </summary>
+ public byte[]? PrecompiledNormalizationData { get; set; }
+
+ /// <summary>
+ /// Represent the vocabulary.
+ /// The list should be sorted by token ID, with entries passed in the order that corresponds to their IDs. In other words,
+ /// the first entry in the list will be mapped to ID 0, the second entry to ID 1, the third to ID 2, and so on.
+ /// Each entry represents a token and its corresponding score.
+ /// </summary>
+ public IEnumerable<(string Token, float Score)>? Vocabulary { get; set; }
+
+ /// <summary>
+ /// The special tokens.
+ /// Special tokens remain intact during encoding and are not split into sub-tokens.
+ /// </summary>
+ public IReadOnlyDictionary<string, int>? SpecialTokens { get; set; }
+ }
+#pragma warning restore MSML_NoInstanceInitializers
+}

src/Microsoft.ML.Tokenizers/Model/SentencePieceTokenizer.cs

@@ -30,6 +30,16 @@ internal SentencePieceTokenizer(ModelProto modelProto, bool addBos, bool addEos,
  };
  }
 
+ internal SentencePieceTokenizer(SentencePieceOptions options)
+ {
+ _model = options.ModelType switch
+ {
+ SentencePieceModelType.Bpe => new SentencePieceBpeModel(options),
+ SentencePieceModelType.Unigram => new SentencePieceUnigramModel(options),
+ _ => throw new ArgumentException($"The model type '{options.ModelType}' is not supported.", nameof(options.ModelType))
+ };
+ }
+
  /// <summary>
  /// The special tokens.
  /// </summary>
@@ -457,5 +467,19 @@ public static SentencePieceTokenizer Create(
 
  return new SentencePieceTokenizer(modelProto, addBeginOfSentence, addEndOfSentence, specialTokens);
  }
+
+ /// <summary>
+ /// Creates an instance of SentencePieceTokenizer.
+ /// </summary>
+ /// <param name="options">The options to use for the sentence piece tokenizer.</param>
+ public static SentencePieceTokenizer Create(SentencePieceOptions options)
+ {
+ if (options is null)
+ {
+ throw new ArgumentNullException(nameof(options));
+ }
+
+ return new SentencePieceTokenizer(options);
+ }
  }
 }