Add in ability to have pre-defined weights for ngrams.#6458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
michaelgsharp merged 5 commits into from
Nov 23, 2022
Merged

Add in ability to have pre-defined weights for ngrams. #6458

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b91fe6f
ngram pre defined weights
michaelgsharp Nov 17, 2022
0bed831
removed commented code
michaelgsharp Nov 17, 2022
a436e61
formatting fix
michaelgsharp Nov 17, 2022
88d7557
renegerate entry points
michaelgsharp Nov 17, 2022
61044f5
fixing API
michaelgsharp Nov 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions src/Microsoft.ML.Transforms/Text/TextCatalog.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -362,6 +362,31 @@ public static WordBagEstimator ProduceWordBags(this TransformsCatalog.TextTransf
=> new WordBagEstimator(Contracts.CheckRef(catalog, nameof(catalog)).GetEnvironment(),
outputColumnName, inputColumnName, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting);

/// <summary>
/// Create a <see cref="WordBagEstimator"/>, which maps the column specified in <paramref name="inputColumnName"/>
/// to a vector of n-gram counts in a new column named <paramref name="outputColumnName"/>.
/// </summary>
/// <remarks>
/// <see cref="WordBagEstimator"/> is different from <see cref="NgramExtractingEstimator"/> in that the former
/// tokenizes text internally and the latter takes tokenized text as input.
/// </remarks>
/// <param name="catalog">The transform's catalog.</param>
/// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.
/// This column's data type will be known-size vector of <see cref="System.Single"/>.</param>
/// <param name="inputColumnName">Name of the column to take the data from.
/// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
/// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
/// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
/// This estimator operates over vector of text.</param>
public static WordBagEstimator ProduceWordBags(this TransformsCatalog.TextTransforms catalog,
string outputColumnName,
char termSeparator,
char freqSeparator,
string inputColumnName = null,
int maximumNgramsCount = NgramExtractingEstimator.Defaults.MaximumNgramsCount)
=> new WordBagEstimator(Contracts.CheckRef(catalog, nameof(catalog)).GetEnvironment(),
outputColumnName, inputColumnName, 1, 0, true, maximumNgramsCount, NgramExtractingEstimator.WeightingCriteria.Tf, termSeparator: termSeparator, freqSeparator: freqSeparator);

/// <summary>
/// Create a <see cref="WordBagEstimator"/>, which maps the multiple columns specified in <paramref name="inputColumnNames"/>
/// to a vector of n-gram counts in a new column named <paramref name="outputColumnName"/>.
Expand Down
200 changes: 198 additions & 2 deletions src/Microsoft.ML.Transforms/Text/WordBagTransform.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
Expand All@@ -12,6 +13,7 @@
using Microsoft.ML.Internal.Utilities;
using Microsoft.ML.Runtime;
using Microsoft.ML.Transforms.Text;
using static Microsoft.ML.Transforms.Text.WordBagBuildingTransformer;

[assembly: LoadableClass(WordBagBuildingTransformer.Summary, typeof(IDataTransform), typeof(WordBagBuildingTransformer), typeof(WordBagBuildingTransformer.Options), typeof(SignatureDataTransform),
"Word Bag Transform", "WordBagTransform", "WordBag")]
Expand All@@ -21,6 +23,16 @@

[assembly: EntryPointModule(typeof(NgramExtractorTransform.NgramExtractorArguments))]

// These are for the internal only TextExpandingTransformer. Not exposed publically
[assembly: LoadableClass(TextExpandingTransformer.Summary, typeof(IDataTransform), typeof(TextExpandingTransformer), null, typeof(SignatureLoadDataTransform),
TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

[assembly: LoadableClass(typeof(TextExpandingTransformer), null, typeof(SignatureLoadModel),
TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

[assembly: LoadableClass(typeof(IRowMapper), typeof(TextExpandingTransformer), null, typeof(SignatureLoadRowMapper),
TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

namespace Microsoft.ML.Transforms.Text
{
/// <summary>
Expand DownExpand Up@@ -144,18 +156,195 @@ internal static IEstimator<ITransformer> CreateEstimator(IHostEnvironment env, O
NgramLength = column.NgramLength,
SkipLength = column.SkipLength,
Weighting = column.Weighting,
UseAllLengths = column.UseAllLengths
UseAllLengths = column.UseAllLengths,
};
}

IEstimator<ITransformer> estimator = NgramExtractionUtils.GetConcatEstimator(h, options.Columns);
estimator = estimator.Append(new WordTokenizingEstimator(env, tokenizeColumns));
if (options.FreqSeparator != default)
{
estimator = estimator.Append(new TextExpandingEstimator(h, tokenizeColumns[0].InputColumnName, options.FreqSeparator, options.TermSeparator));
}
estimator = estimator.Append(new WordTokenizingEstimator(h, tokenizeColumns));
estimator = estimator.Append(NgramExtractorTransform.CreateEstimator(h, extractorArgs, estimator.GetOutputSchema(inputSchema)));
return estimator;
}

internal static IDataTransform Create(IHostEnvironment env, Options options, IDataView input) =>
(IDataTransform)CreateEstimator(env, options, SchemaShape.Create(input.Schema)).Fit(input).Transform(input);

#region TextExpander

// Internal only estimator used to facilitate the expansion of ngrams with pre-defined weights
internal sealed class TextExpandingEstimator : TrivialEstimator<TextExpandingTransformer>
{
private readonly string _columnName;
public TextExpandingEstimator(IHostEnvironment env, string columnName, char freqSeparator, char termSeparator)
: base(Contracts.CheckRef(env, nameof(env)).Register(nameof(TextExpandingEstimator)), new TextExpandingTransformer(env, columnName, freqSeparator, termSeparator))
{
_columnName = columnName;
}

public override SchemaShape GetOutputSchema(SchemaShape inputSchema)
{
Host.CheckValue(inputSchema, nameof(inputSchema));
if (!inputSchema.TryFindColumn(_columnName, out SchemaShape.Column outCol) && outCol.ItemType != TextDataViewType.Instance)
{
throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", _columnName);
}

return inputSchema;
}
}

// Internal only transformer used to facilitate the expansion of ngrams with pre-defined weights
internal sealed class TextExpandingTransformer : RowToRowTransformerBase
{
internal const string Summary = "Expands text in the format of term:freq; to have the correct number of terms";
internal const string UserName = "Text Expanding Transform";
internal const string LoadName = "TextExpand";

internal const string LoaderSignature = "TextExpandTransform";

private readonly string _columnName;
private readonly char _freqSeparator;
private readonly char _termSeparator;

public TextExpandingTransformer(IHostEnvironment env, string columnName, char freqSeparator, char termSeparator)
: base(Contracts.CheckRef(env, nameof(env)).Register(nameof(TextExpandingTransformer)))
{
_columnName = columnName;
_freqSeparator = freqSeparator;
_termSeparator = termSeparator;
}

private static VersionInfo GetVersionInfo()
{
return new VersionInfo(
modelSignature: "TEXT EXP",
verWrittenCur: 0x00010001, // Initial
verReadableCur: 0x00010001,
verWeCanReadBack: 0x00010001,
loaderSignature: LoaderSignature,
loaderAssemblyName: typeof(TextExpandingTransformer).Assembly.FullName);
}

/// <summary>
/// Factory method for SignatureLoadModel.
/// </summary>
private TextExpandingTransformer(IHostEnvironment env, ModelLoadContext ctx) :
base(Contracts.CheckRef(env, nameof(env)).Register(nameof(ColumnConcatenatingTransformer)))
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel(GetVersionInfo());
// *** Binary format ***
// string: column n ame
// char: frequency separator
// char: term separator

_columnName = ctx.Reader.ReadString();
_freqSeparator = ctx.Reader.ReadChar();
_termSeparator = ctx.Reader.ReadChar();
}

/// <summary>
/// Factory method for SignatureLoadRowMapper.
/// </summary>
private static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, DataViewSchema inputSchema)
=> new TextExpandingTransformer(env, ctx).MakeRowMapper(inputSchema);

/// <summary>
/// Factory method for SignatureLoadDataTransform.
/// </summary>
private static IDataTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
=> new TextExpandingTransformer(env, ctx).MakeDataTransform(input);

private protected override IRowMapper MakeRowMapper(DataViewSchema schema)
{
return new Mapper(Host, schema, this);
}

private protected override void SaveModel(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());

// *** Binary format ***
// string: column n ame
// char: frequency separator
// char: term separator

ctx.Writer.Write(_columnName);
ctx.Writer.Write(_freqSeparator);
ctx.Writer.Write(_termSeparator);
}

private sealed class Mapper : MapperBase
{
private readonly TextExpandingTransformer _parent;
public Mapper(IHost host, DataViewSchema inputSchema, RowToRowTransformerBase parent)
: base(host, inputSchema, parent)
{
_parent = (TextExpandingTransformer)parent;
}

protected override DataViewSchema.DetachedColumn[] GetOutputColumnsCore()
{
return new DataViewSchema.DetachedColumn[]
{
new DataViewSchema.DetachedColumn(_parent._columnName, TextDataViewType.Instance)
};
}

protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func<int, bool> activeOutput, out Action disposer)
{
disposer = null;
ValueGetter<ReadOnlyMemory<char>> srcGetter = input.GetGetter<ReadOnlyMemory<char>>(input.Schema.GetColumnOrNull(_parent._columnName).Value);
ReadOnlyMemory<char> inputMem = default;
var sb = new StringBuilder();

ValueGetter<ReadOnlyMemory<char>> result = (ref ReadOnlyMemory<char> dst) =>
{
sb.Clear();
srcGetter(ref inputMem);
var inputText = inputMem.ToString();
foreach (var termFreq in inputText.Split(_parent._termSeparator))
{
var tf = termFreq.Split(_parent._freqSeparator);
if (tf.Length != 2)
sb.Append(tf[0] + " ");
else
{
for (int i = 0; i < int.Parse(tf[1]); i++)
sb.Append(tf[0] + " ");
}
}

dst = sb.ToString().AsMemory();
};

return result;
}

private protected override Func<int, bool> GetDependenciesCore(Func<int, bool> activeOutput)
{
var active = new bool[InputSchema.Count];
if (activeOutput(0))
{
active[InputSchema.GetColumnOrNull(_parent._columnName).Value.Index] = true;
}
return col => active[col];
}

private protected override void SaveModel(ModelSaveContext ctx)
{
_parent.SaveModel(ctx);
}
}
}

#endregion TextExpander
}

/// <summary>
Expand DownExpand Up@@ -235,6 +424,13 @@ internal abstract class ArgumentsBase

[Argument(ArgumentType.AtMostOnce, HelpText = "The weighting criteria")]
public NgramExtractingEstimator.WeightingCriteria Weighting = NgramExtractingEstimator.Defaults.Weighting;

[Argument(ArgumentType.AtMostOnce, HelpText = "Separator used to separate terms/frequency pairs.")]
public char TermSeparator = default;

[Argument(ArgumentType.AtMostOnce, HelpText = "Separator used to separate terms from their frequency.")]
public char FreqSeparator = default;

}

[TlcModule.Component(Name = "NGram", FriendlyName = "NGram Extractor Transform", Alias = "NGramExtractorTransform,NGramExtractor",
Expand Down
30 changes: 24 additions & 6 deletions src/Microsoft.ML.Transforms/Text/WrappedTextTransformers.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -45,6 +45,8 @@ public sealed class WordBagEstimator : IEstimator<ITransformer>
private readonly bool _useAllLengths;
private readonly int _maxNumTerms;
private readonly NgramExtractingEstimator.WeightingCriteria _weighting;
private readonly char _termSeparator;
private readonly char _freqSeparator;

/// <summary>
/// Options for how the n-grams are extracted.
Expand DownExpand Up@@ -99,15 +101,19 @@ public Options()
/// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
/// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
/// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
/// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
/// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
internal WordBagEstimator(IHostEnvironment env,
string outputColumnName,
string inputColumnName = null,
int ngramLength = 1,
int skipLength = 0,
bool useAllLengths = true,
int maximumNgramsCount = 10000000,
NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
: this(env, outputColumnName, new[] { inputColumnName ?? outputColumnName }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting)
NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
char termSeparator = default,
char freqSeparator = default)
: this(env, outputColumnName, new[] { inputColumnName ?? outputColumnName }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting, termSeparator, freqSeparator)
{
}

Expand All@@ -123,15 +129,19 @@ internal WordBagEstimator(IHostEnvironment env,
/// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
/// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
/// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
/// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
/// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
internal WordBagEstimator(IHostEnvironment env,
string outputColumnName,
string[] inputColumnNames,
int ngramLength = 1,
int skipLength = 0,
bool useAllLengths = true,
int maximumNgramsCount = 10000000,
NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
: this(env, new[] { (outputColumnName, inputColumnNames) }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting)
NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
char termSeparator = default,
char freqSeparator = default)
: this(env, new[] { (outputColumnName, inputColumnNames) }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting, termSeparator, freqSeparator)
{
}

Expand All@@ -146,13 +156,17 @@ internal WordBagEstimator(IHostEnvironment env,
/// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
/// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
/// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
/// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
/// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
internal WordBagEstimator(IHostEnvironment env,
(string outputColumnName, string[] inputColumnNames)[] columns,
int ngramLength = 1,
int skipLength = 0,
bool useAllLengths = true,
int maximumNgramsCount = 10000000,
NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
char termSeparator = default,
char freqSeparator = default)
{
Contracts.CheckValue(env, nameof(env));
_host = env.Register(nameof(WordBagEstimator));
Expand All@@ -169,6 +183,8 @@ internal WordBagEstimator(IHostEnvironment env,
_useAllLengths = useAllLengths;
_maxNumTerms = maximumNgramsCount;
_weighting = weighting;
_termSeparator = termSeparator;
_freqSeparator = freqSeparator;
}

/// <summary> Trains and returns a <see cref="ITransformer"/>.</summary>
Expand All@@ -187,7 +203,9 @@ private WordBagBuildingTransformer.Options CreateOptions()
SkipLength = _skipLength,
UseAllLengths = _useAllLengths,
MaxNumTerms = new[] { _maxNumTerms },
Weighting = _weighting
Weighting = _weighting,
TermSeparator = _termSeparator,
FreqSeparator = _freqSeparator,
};
}

Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Add in ability to have pre-defined weights for ngrams.#6458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

michaelgsharp
merged 5 commits into

from

Nov 23, 2022
 Merged


Add in ability to have pre-defined weights for ngrams.
#6458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
b91fe6f

ngram pre defined weights


michaelgsharp Nov 17, 2022
0bed831

removed commented code


michaelgsharp Nov 17, 2022
a436e61

formatting fix


michaelgsharp Nov 17, 2022
88d7557

renegerate entry points


michaelgsharp Nov 17, 2022
61044f5

fixing API


michaelgsharp Nov 23, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


25 changes: 25 additions & 0 deletions
src/Microsoft.ML.Transforms/Text/TextCatalog.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -362,6 +362,31 @@ public static WordBagEstimator ProduceWordBags(this TransformsCatalog.TextTransf
 => new WordBagEstimator(Contracts.CheckRef(catalog, nameof(catalog)).GetEnvironment(),
 outputColumnName, inputColumnName, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting);

 /// <summary>
 /// Create a <see cref="WordBagEstimator"/>, which maps the column specified in <paramref name="inputColumnName"/>
 /// to a vector of n-gram counts in a new column named <paramref name="outputColumnName"/>.
 /// </summary>
 /// <remarks>
 /// <see cref="WordBagEstimator"/> is different from <see cref="NgramExtractingEstimator"/> in that the former
 /// tokenizes text internally and the latter takes tokenized text as input.
 /// </remarks>
 /// <param name="catalog">The transform's catalog.</param>
 /// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.
 /// This column's data type will be known-size vector of <see cref="System.Single"/>.</param>
 /// <param name="inputColumnName">Name of the column to take the data from.
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 /// This estimator operates over vector of text.</param>
 public static WordBagEstimator ProduceWordBags(this TransformsCatalog.TextTransforms catalog,
 string outputColumnName,
 char termSeparator,
 char freqSeparator,
 string inputColumnName = null,
 int maximumNgramsCount = NgramExtractingEstimator.Defaults.MaximumNgramsCount)
 => new WordBagEstimator(Contracts.CheckRef(catalog, nameof(catalog)).GetEnvironment(),
 outputColumnName, inputColumnName, 1, 0, true, maximumNgramsCount, NgramExtractingEstimator.WeightingCriteria.Tf, termSeparator: termSeparator, freqSeparator: freqSeparator);

 /// <summary>
 /// Create a <see cref="WordBagEstimator"/>, which maps the multiple columns specified in <paramref name="inputColumnNames"/>
 /// to a vector of n-gram counts in a new column named <paramref name="outputColumnName"/>.
Expand Down

200 changes: 198 additions & 2 deletions
src/Microsoft.ML.Transforms/Text/WordBagTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
Expand All@@ -12,6 +13,7 @@
using Microsoft.ML.Internal.Utilities;
using Microsoft.ML.Runtime;
using Microsoft.ML.Transforms.Text;
using static Microsoft.ML.Transforms.Text.WordBagBuildingTransformer;

[assembly: LoadableClass(WordBagBuildingTransformer.Summary, typeof(IDataTransform), typeof(WordBagBuildingTransformer), typeof(WordBagBuildingTransformer.Options), typeof(SignatureDataTransform),
 "Word Bag Transform", "WordBagTransform", "WordBag")]
Expand All@@ -21,6 +23,16 @@

[assembly: EntryPointModule(typeof(NgramExtractorTransform.NgramExtractorArguments))]

// These are for the internal only TextExpandingTransformer. Not exposed publically
[assembly: LoadableClass(TextExpandingTransformer.Summary, typeof(IDataTransform), typeof(TextExpandingTransformer), null, typeof(SignatureLoadDataTransform),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

[assembly: LoadableClass(typeof(TextExpandingTransformer), null, typeof(SignatureLoadModel),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

[assembly: LoadableClass(typeof(IRowMapper), typeof(TextExpandingTransformer), null, typeof(SignatureLoadRowMapper),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

namespace Microsoft.ML.Transforms.Text
{
 /// <summary>
Expand DownExpand Up@@ -144,18 +156,195 @@ internal static IEstimator<ITransformer> CreateEstimator(IHostEnvironment env, O
 NgramLength = column.NgramLength,
 SkipLength = column.SkipLength,
 Weighting = column.Weighting,
 UseAllLengths = column.UseAllLengths
 UseAllLengths = column.UseAllLengths,
 };
 }

 IEstimator<ITransformer> estimator = NgramExtractionUtils.GetConcatEstimator(h, options.Columns);
 estimator = estimator.Append(new WordTokenizingEstimator(env, tokenizeColumns));
 if (options.FreqSeparator != default)
 {
 estimator = estimator.Append(new TextExpandingEstimator(h, tokenizeColumns[0].InputColumnName, options.FreqSeparator, options.TermSeparator));
 }
 estimator = estimator.Append(new WordTokenizingEstimator(h, tokenizeColumns));
 estimator = estimator.Append(NgramExtractorTransform.CreateEstimator(h, extractorArgs, estimator.GetOutputSchema(inputSchema)));
 return estimator;
 }

 internal static IDataTransform Create(IHostEnvironment env, Options options, IDataView input) =>
 (IDataTransform)CreateEstimator(env, options, SchemaShape.Create(input.Schema)).Fit(input).Transform(input);

 #region TextExpander

 // Internal only estimator used to facilitate the expansion of ngrams with pre-defined weights
 internal sealed class TextExpandingEstimator : TrivialEstimator<TextExpandingTransformer>
 {
 private readonly string _columnName;
 public TextExpandingEstimator(IHostEnvironment env, string columnName, char freqSeparator, char termSeparator)
 : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(TextExpandingEstimator)), new TextExpandingTransformer(env, columnName, freqSeparator, termSeparator))
 {
 _columnName = columnName;
 }

 public override SchemaShape GetOutputSchema(SchemaShape inputSchema)
 {
 Host.CheckValue(inputSchema, nameof(inputSchema));
 if (!inputSchema.TryFindColumn(_columnName, out SchemaShape.Column outCol) && outCol.ItemType != TextDataViewType.Instance)
 {
 throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", _columnName);
 }

 return inputSchema;
 }
 }

 // Internal only transformer used to facilitate the expansion of ngrams with pre-defined weights
 internal sealed class TextExpandingTransformer : RowToRowTransformerBase
 {
 internal const string Summary = "Expands text in the format of term:freq; to have the correct number of terms";
 internal const string UserName = "Text Expanding Transform";
 internal const string LoadName = "TextExpand";

 internal const string LoaderSignature = "TextExpandTransform";

 private readonly string _columnName;
 private readonly char _freqSeparator;
 private readonly char _termSeparator;

 public TextExpandingTransformer(IHostEnvironment env, string columnName, char freqSeparator, char termSeparator)
 : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(TextExpandingTransformer)))
 {
 _columnName = columnName;
 _freqSeparator = freqSeparator;
 _termSeparator = termSeparator;
 }

 private static VersionInfo GetVersionInfo()
 {
 return new VersionInfo(
 modelSignature: "TEXT EXP",
 verWrittenCur: 0x00010001, // Initial
 verReadableCur: 0x00010001,
 verWeCanReadBack: 0x00010001,
 loaderSignature: LoaderSignature,
 loaderAssemblyName: typeof(TextExpandingTransformer).Assembly.FullName);
 }

 /// <summary>
 /// Factory method for SignatureLoadModel.
 /// </summary>
 private TextExpandingTransformer(IHostEnvironment env, ModelLoadContext ctx) :
 base(Contracts.CheckRef(env, nameof(env)).Register(nameof(ColumnConcatenatingTransformer)))
 {
 Host.CheckValue(ctx, nameof(ctx));
 ctx.CheckAtModel(GetVersionInfo());
 // *** Binary format ***
 // string: column n ame
 // char: frequency separator
 // char: term separator

 _columnName = ctx.Reader.ReadString();
 _freqSeparator = ctx.Reader.ReadChar();
 _termSeparator = ctx.Reader.ReadChar();
 }

 /// <summary>
 /// Factory method for SignatureLoadRowMapper.
 /// </summary>
 private static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, DataViewSchema inputSchema)
 => new TextExpandingTransformer(env, ctx).MakeRowMapper(inputSchema);

 /// <summary>
 /// Factory method for SignatureLoadDataTransform.
 /// </summary>
 private static IDataTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
 => new TextExpandingTransformer(env, ctx).MakeDataTransform(input);

 private protected override IRowMapper MakeRowMapper(DataViewSchema schema)
 {
 return new Mapper(Host, schema, this);
 }

 private protected override void SaveModel(ModelSaveContext ctx)
 {
 Host.CheckValue(ctx, nameof(ctx));
 ctx.CheckAtModel();
 ctx.SetVersionInfo(GetVersionInfo());

 // *** Binary format ***
 // string: column n ame
 // char: frequency separator
 // char: term separator

 ctx.Writer.Write(_columnName);
 ctx.Writer.Write(_freqSeparator);
 ctx.Writer.Write(_termSeparator);
 }

 private sealed class Mapper : MapperBase
 {
 private readonly TextExpandingTransformer _parent;
 public Mapper(IHost host, DataViewSchema inputSchema, RowToRowTransformerBase parent)
 : base(host, inputSchema, parent)
 {
 _parent = (TextExpandingTransformer)parent;
 }

 protected override DataViewSchema.DetachedColumn[] GetOutputColumnsCore()
 {
 return new DataViewSchema.DetachedColumn[]
 {
 new DataViewSchema.DetachedColumn(_parent._columnName, TextDataViewType.Instance)
 };
 }

 protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func<int, bool> activeOutput, out Action disposer)
 {
 disposer = null;
 ValueGetter<ReadOnlyMemory<char>> srcGetter = input.GetGetter<ReadOnlyMemory<char>>(input.Schema.GetColumnOrNull(_parent._columnName).Value);
 ReadOnlyMemory<char> inputMem = default;
 var sb = new StringBuilder();

 ValueGetter<ReadOnlyMemory<char>> result = (ref ReadOnlyMemory<char> dst) =>
 {
 sb.Clear();
 srcGetter(ref inputMem);
 var inputText = inputMem.ToString();
 foreach (var termFreq in inputText.Split(_parent._termSeparator))
 {
 var tf = termFreq.Split(_parent._freqSeparator);
 if (tf.Length != 2)
 sb.Append(tf[0] + " ");
 else
 {
 for (int i = 0; i < int.Parse(tf[1]); i++)
 sb.Append(tf[0] + " ");
 }
 }

 dst = sb.ToString().AsMemory();
 };

 return result;
 }

 private protected override Func<int, bool> GetDependenciesCore(Func<int, bool> activeOutput)
 {
 var active = new bool[InputSchema.Count];
 if (activeOutput(0))
 {
 active[InputSchema.GetColumnOrNull(_parent._columnName).Value.Index] = true;
 }
 return col => active[col];
 }

 private protected override void SaveModel(ModelSaveContext ctx)
 {
 _parent.SaveModel(ctx);
 }
 }
 }

 #endregion TextExpander
 }

 /// <summary>
Expand DownExpand Up@@ -235,6 +424,13 @@ internal abstract class ArgumentsBase

 [Argument(ArgumentType.AtMostOnce, HelpText = "The weighting criteria")]
 public NgramExtractingEstimator.WeightingCriteria Weighting = NgramExtractingEstimator.Defaults.Weighting;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Separator used to separate terms/frequency pairs.")]
 public char TermSeparator = default;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Separator used to separate terms from their frequency.")]
 public char FreqSeparator = default;

 }

 [TlcModule.Component(Name = "NGram", FriendlyName = "NGram Extractor Transform", Alias = "NGramExtractorTransform,NGramExtractor",
Expand Down

30 changes: 24 additions & 6 deletions
src/Microsoft.ML.Transforms/Text/WrappedTextTransformers.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,6 +45,8 @@ public sealed class WordBagEstimator : IEstimator<ITransformer>
 private readonly bool _useAllLengths;
 private readonly int _maxNumTerms;
 private readonly NgramExtractingEstimator.WeightingCriteria _weighting;
 private readonly char _termSeparator;
 private readonly char _freqSeparator;

 /// <summary>
 /// Options for how the n-grams are extracted.
Expand DownExpand Up@@ -99,15 +101,19 @@ public Options()
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 string outputColumnName,
 string inputColumnName = null,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 : this(env, outputColumnName, new[] { inputColumnName ?? outputColumnName }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 : this(env, outputColumnName, new[] { inputColumnName ?? outputColumnName }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting, termSeparator, freqSeparator)
 {
 }

Expand All@@ -123,15 +129,19 @@ internal WordBagEstimator(IHostEnvironment env,
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 string outputColumnName,
 string[] inputColumnNames,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 : this(env, new[] { (outputColumnName, inputColumnNames) }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 : this(env, new[] { (outputColumnName, inputColumnNames) }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting, termSeparator, freqSeparator)
 {
 }

Expand All@@ -146,13 +156,17 @@ internal WordBagEstimator(IHostEnvironment env,
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 (string outputColumnName, string[] inputColumnNames)[] columns,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 {
 Contracts.CheckValue(env, nameof(env));
 _host = env.Register(nameof(WordBagEstimator));
Expand All@@ -169,6 +183,8 @@ internal WordBagEstimator(IHostEnvironment env,
 _useAllLengths = useAllLengths;
 _maxNumTerms = maximumNgramsCount;
 _weighting = weighting;
 _termSeparator = termSeparator;
 _freqSeparator = freqSeparator;
 }

 /// <summary> Trains and returns a <see cref="ITransformer"/>.</summary>
Expand All@@ -187,7 +203,9 @@ private WordBagBuildingTransformer.Options CreateOptions()
 SkipLength = _skipLength,
 UseAllLengths = _useAllLengths,
 MaxNumTerms = new[] { _maxNumTerms },
 Weighting = _weighting
 Weighting = _weighting,
 TermSeparator = _termSeparator,
 FreqSeparator = _freqSeparator,
 };
 }

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Add in ability to have pre-defined weights for ngrams.#6458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

michaelgsharp
merged 5 commits into

from

Nov 23, 2022
 Merged


Add in ability to have pre-defined weights for ngrams.
#6458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
b91fe6f

ngram pre defined weights


michaelgsharp Nov 17, 2022
0bed831

removed commented code


michaelgsharp Nov 17, 2022
a436e61

formatting fix


michaelgsharp Nov 17, 2022
88d7557

renegerate entry points


michaelgsharp Nov 17, 2022
61044f5

fixing API


michaelgsharp Nov 23, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


25 changes: 25 additions & 0 deletions
src/Microsoft.ML.Transforms/Text/TextCatalog.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -362,6 +362,31 @@ public static WordBagEstimator ProduceWordBags(this TransformsCatalog.TextTransf
 => new WordBagEstimator(Contracts.CheckRef(catalog, nameof(catalog)).GetEnvironment(),
 outputColumnName, inputColumnName, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting);

 /// <summary>
 /// Create a <see cref="WordBagEstimator"/>, which maps the column specified in <paramref name="inputColumnName"/>
 /// to a vector of n-gram counts in a new column named <paramref name="outputColumnName"/>.
 /// </summary>
 /// <remarks>
 /// <see cref="WordBagEstimator"/> is different from <see cref="NgramExtractingEstimator"/> in that the former
 /// tokenizes text internally and the latter takes tokenized text as input.
 /// </remarks>
 /// <param name="catalog">The transform's catalog.</param>
 /// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.
 /// This column's data type will be known-size vector of <see cref="System.Single"/>.</param>
 /// <param name="inputColumnName">Name of the column to take the data from.
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 /// This estimator operates over vector of text.</param>
 public static WordBagEstimator ProduceWordBags(this TransformsCatalog.TextTransforms catalog,
 string outputColumnName,
 char termSeparator,
 char freqSeparator,
 string inputColumnName = null,
 int maximumNgramsCount = NgramExtractingEstimator.Defaults.MaximumNgramsCount)
 => new WordBagEstimator(Contracts.CheckRef(catalog, nameof(catalog)).GetEnvironment(),
 outputColumnName, inputColumnName, 1, 0, true, maximumNgramsCount, NgramExtractingEstimator.WeightingCriteria.Tf, termSeparator: termSeparator, freqSeparator: freqSeparator);

 /// <summary>
 /// Create a <see cref="WordBagEstimator"/>, which maps the multiple columns specified in <paramref name="inputColumnNames"/>
 /// to a vector of n-gram counts in a new column named <paramref name="outputColumnName"/>.
Expand Down

200 changes: 198 additions & 2 deletions
src/Microsoft.ML.Transforms/Text/WordBagTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
Expand All@@ -12,6 +13,7 @@
using Microsoft.ML.Internal.Utilities;
using Microsoft.ML.Runtime;
using Microsoft.ML.Transforms.Text;
using static Microsoft.ML.Transforms.Text.WordBagBuildingTransformer;

[assembly: LoadableClass(WordBagBuildingTransformer.Summary, typeof(IDataTransform), typeof(WordBagBuildingTransformer), typeof(WordBagBuildingTransformer.Options), typeof(SignatureDataTransform),
 "Word Bag Transform", "WordBagTransform", "WordBag")]
Expand All@@ -21,6 +23,16 @@

[assembly: EntryPointModule(typeof(NgramExtractorTransform.NgramExtractorArguments))]

// These are for the internal only TextExpandingTransformer. Not exposed publically
[assembly: LoadableClass(TextExpandingTransformer.Summary, typeof(IDataTransform), typeof(TextExpandingTransformer), null, typeof(SignatureLoadDataTransform),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

[assembly: LoadableClass(typeof(TextExpandingTransformer), null, typeof(SignatureLoadModel),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

[assembly: LoadableClass(typeof(IRowMapper), typeof(TextExpandingTransformer), null, typeof(SignatureLoadRowMapper),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

namespace Microsoft.ML.Transforms.Text
{
 /// <summary>
Expand DownExpand Up@@ -144,18 +156,195 @@ internal static IEstimator<ITransformer> CreateEstimator(IHostEnvironment env, O
 NgramLength = column.NgramLength,
 SkipLength = column.SkipLength,
 Weighting = column.Weighting,
 UseAllLengths = column.UseAllLengths
 UseAllLengths = column.UseAllLengths,
 };
 }

 IEstimator<ITransformer> estimator = NgramExtractionUtils.GetConcatEstimator(h, options.Columns);
 estimator = estimator.Append(new WordTokenizingEstimator(env, tokenizeColumns));
 if (options.FreqSeparator != default)
 {
 estimator = estimator.Append(new TextExpandingEstimator(h, tokenizeColumns[0].InputColumnName, options.FreqSeparator, options.TermSeparator));
 }
 estimator = estimator.Append(new WordTokenizingEstimator(h, tokenizeColumns));
 estimator = estimator.Append(NgramExtractorTransform.CreateEstimator(h, extractorArgs, estimator.GetOutputSchema(inputSchema)));
 return estimator;
 }

 internal static IDataTransform Create(IHostEnvironment env, Options options, IDataView input) =>
 (IDataTransform)CreateEstimator(env, options, SchemaShape.Create(input.Schema)).Fit(input).Transform(input);

 #region TextExpander

 // Internal only estimator used to facilitate the expansion of ngrams with pre-defined weights
 internal sealed class TextExpandingEstimator : TrivialEstimator<TextExpandingTransformer>
 {
 private readonly string _columnName;
 public TextExpandingEstimator(IHostEnvironment env, string columnName, char freqSeparator, char termSeparator)
 : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(TextExpandingEstimator)), new TextExpandingTransformer(env, columnName, freqSeparator, termSeparator))
 {
 _columnName = columnName;
 }

 public override SchemaShape GetOutputSchema(SchemaShape inputSchema)
 {
 Host.CheckValue(inputSchema, nameof(inputSchema));
 if (!inputSchema.TryFindColumn(_columnName, out SchemaShape.Column outCol) && outCol.ItemType != TextDataViewType.Instance)
 {
 throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", _columnName);
 }

 return inputSchema;
 }
 }

 // Internal only transformer used to facilitate the expansion of ngrams with pre-defined weights
 internal sealed class TextExpandingTransformer : RowToRowTransformerBase
 {
 internal const string Summary = "Expands text in the format of term:freq; to have the correct number of terms";
 internal const string UserName = "Text Expanding Transform";
 internal const string LoadName = "TextExpand";

 internal const string LoaderSignature = "TextExpandTransform";

 private readonly string _columnName;
 private readonly char _freqSeparator;
 private readonly char _termSeparator;

 public TextExpandingTransformer(IHostEnvironment env, string columnName, char freqSeparator, char termSeparator)
 : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(TextExpandingTransformer)))
 {
 _columnName = columnName;
 _freqSeparator = freqSeparator;
 _termSeparator = termSeparator;
 }

 private static VersionInfo GetVersionInfo()
 {
 return new VersionInfo(
 modelSignature: "TEXT EXP",
 verWrittenCur: 0x00010001, // Initial
 verReadableCur: 0x00010001,
 verWeCanReadBack: 0x00010001,
 loaderSignature: LoaderSignature,
 loaderAssemblyName: typeof(TextExpandingTransformer).Assembly.FullName);
 }

 /// <summary>
 /// Factory method for SignatureLoadModel.
 /// </summary>
 private TextExpandingTransformer(IHostEnvironment env, ModelLoadContext ctx) :
 base(Contracts.CheckRef(env, nameof(env)).Register(nameof(ColumnConcatenatingTransformer)))
 {
 Host.CheckValue(ctx, nameof(ctx));
 ctx.CheckAtModel(GetVersionInfo());
 // *** Binary format ***
 // string: column n ame
 // char: frequency separator
 // char: term separator

 _columnName = ctx.Reader.ReadString();
 _freqSeparator = ctx.Reader.ReadChar();
 _termSeparator = ctx.Reader.ReadChar();
 }

 /// <summary>
 /// Factory method for SignatureLoadRowMapper.
 /// </summary>
 private static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, DataViewSchema inputSchema)
 => new TextExpandingTransformer(env, ctx).MakeRowMapper(inputSchema);

 /// <summary>
 /// Factory method for SignatureLoadDataTransform.
 /// </summary>
 private static IDataTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
 => new TextExpandingTransformer(env, ctx).MakeDataTransform(input);

 private protected override IRowMapper MakeRowMapper(DataViewSchema schema)
 {
 return new Mapper(Host, schema, this);
 }

 private protected override void SaveModel(ModelSaveContext ctx)
 {
 Host.CheckValue(ctx, nameof(ctx));
 ctx.CheckAtModel();
 ctx.SetVersionInfo(GetVersionInfo());

 // *** Binary format ***
 // string: column n ame
 // char: frequency separator
 // char: term separator

 ctx.Writer.Write(_columnName);
 ctx.Writer.Write(_freqSeparator);
 ctx.Writer.Write(_termSeparator);
 }

 private sealed class Mapper : MapperBase
 {
 private readonly TextExpandingTransformer _parent;
 public Mapper(IHost host, DataViewSchema inputSchema, RowToRowTransformerBase parent)
 : base(host, inputSchema, parent)
 {
 _parent = (TextExpandingTransformer)parent;
 }

 protected override DataViewSchema.DetachedColumn[] GetOutputColumnsCore()
 {
 return new DataViewSchema.DetachedColumn[]
 {
 new DataViewSchema.DetachedColumn(_parent._columnName, TextDataViewType.Instance)
 };
 }

 protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func<int, bool> activeOutput, out Action disposer)
 {
 disposer = null;
 ValueGetter<ReadOnlyMemory<char>> srcGetter = input.GetGetter<ReadOnlyMemory<char>>(input.Schema.GetColumnOrNull(_parent._columnName).Value);
 ReadOnlyMemory<char> inputMem = default;
 var sb = new StringBuilder();

 ValueGetter<ReadOnlyMemory<char>> result = (ref ReadOnlyMemory<char> dst) =>
 {
 sb.Clear();
 srcGetter(ref inputMem);
 var inputText = inputMem.ToString();
 foreach (var termFreq in inputText.Split(_parent._termSeparator))
 {
 var tf = termFreq.Split(_parent._freqSeparator);
 if (tf.Length != 2)
 sb.Append(tf[0] + " ");
 else
 {
 for (int i = 0; i < int.Parse(tf[1]); i++)
 sb.Append(tf[0] + " ");
 }
 }

 dst = sb.ToString().AsMemory();
 };

 return result;
 }

 private protected override Func<int, bool> GetDependenciesCore(Func<int, bool> activeOutput)
 {
 var active = new bool[InputSchema.Count];
 if (activeOutput(0))
 {
 active[InputSchema.GetColumnOrNull(_parent._columnName).Value.Index] = true;
 }
 return col => active[col];
 }

 private protected override void SaveModel(ModelSaveContext ctx)
 {
 _parent.SaveModel(ctx);
 }
 }
 }

 #endregion TextExpander
 }

 /// <summary>
Expand DownExpand Up@@ -235,6 +424,13 @@ internal abstract class ArgumentsBase

 [Argument(ArgumentType.AtMostOnce, HelpText = "The weighting criteria")]
 public NgramExtractingEstimator.WeightingCriteria Weighting = NgramExtractingEstimator.Defaults.Weighting;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Separator used to separate terms/frequency pairs.")]
 public char TermSeparator = default;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Separator used to separate terms from their frequency.")]
 public char FreqSeparator = default;

 }

 [TlcModule.Component(Name = "NGram", FriendlyName = "NGram Extractor Transform", Alias = "NGramExtractorTransform,NGramExtractor",
Expand Down

30 changes: 24 additions & 6 deletions
src/Microsoft.ML.Transforms/Text/WrappedTextTransformers.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,6 +45,8 @@ public sealed class WordBagEstimator : IEstimator<ITransformer>
 private readonly bool _useAllLengths;
 private readonly int _maxNumTerms;
 private readonly NgramExtractingEstimator.WeightingCriteria _weighting;
 private readonly char _termSeparator;
 private readonly char _freqSeparator;

 /// <summary>
 /// Options for how the n-grams are extracted.
Expand DownExpand Up@@ -99,15 +101,19 @@ public Options()
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 string outputColumnName,
 string inputColumnName = null,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 : this(env, outputColumnName, new[] { inputColumnName ?? outputColumnName }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 : this(env, outputColumnName, new[] { inputColumnName ?? outputColumnName }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting, termSeparator, freqSeparator)
 {
 }

Expand All@@ -123,15 +129,19 @@ internal WordBagEstimator(IHostEnvironment env,
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 string outputColumnName,
 string[] inputColumnNames,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 : this(env, new[] { (outputColumnName, inputColumnNames) }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 : this(env, new[] { (outputColumnName, inputColumnNames) }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting, termSeparator, freqSeparator)
 {
 }

Expand All@@ -146,13 +156,17 @@ internal WordBagEstimator(IHostEnvironment env,
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 (string outputColumnName, string[] inputColumnNames)[] columns,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 {
 Contracts.CheckValue(env, nameof(env));
 _host = env.Register(nameof(WordBagEstimator));
Expand All@@ -169,6 +183,8 @@ internal WordBagEstimator(IHostEnvironment env,
 _useAllLengths = useAllLengths;
 _maxNumTerms = maximumNgramsCount;
 _weighting = weighting;
 _termSeparator = termSeparator;
 _freqSeparator = freqSeparator;
 }

 /// <summary> Trains and returns a <see cref="ITransformer"/>.</summary>
Expand All@@ -187,7 +203,9 @@ private WordBagBuildingTransformer.Options CreateOptions()
 SkipLength = _skipLength,
 UseAllLengths = _useAllLengths,
 MaxNumTerms = new[] { _maxNumTerms },
 Weighting = _weighting
 Weighting = _weighting,
 TermSeparator = _termSeparator,
 FreqSeparator = _freqSeparator,
 };
 }

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Add in ability to have pre-defined weights for ngrams.#6458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

michaelgsharp
merged 5 commits into

from

Nov 23, 2022
 Merged


Add in ability to have pre-defined weights for ngrams.
#6458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
b91fe6f

ngram pre defined weights


michaelgsharp Nov 17, 2022
0bed831

removed commented code


michaelgsharp Nov 17, 2022
a436e61

formatting fix


michaelgsharp Nov 17, 2022
88d7557

renegerate entry points


michaelgsharp Nov 17, 2022
61044f5

fixing API


michaelgsharp Nov 23, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


25 changes: 25 additions & 0 deletions
src/Microsoft.ML.Transforms/Text/TextCatalog.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -362,6 +362,31 @@ public static WordBagEstimator ProduceWordBags(this TransformsCatalog.TextTransf
 => new WordBagEstimator(Contracts.CheckRef(catalog, nameof(catalog)).GetEnvironment(),
 outputColumnName, inputColumnName, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting);

 /// <summary>
 /// Create a <see cref="WordBagEstimator"/>, which maps the column specified in <paramref name="inputColumnName"/>
 /// to a vector of n-gram counts in a new column named <paramref name="outputColumnName"/>.
 /// </summary>
 /// <remarks>
 /// <see cref="WordBagEstimator"/> is different from <see cref="NgramExtractingEstimator"/> in that the former
 /// tokenizes text internally and the latter takes tokenized text as input.
 /// </remarks>
 /// <param name="catalog">The transform's catalog.</param>
 /// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.
 /// This column's data type will be known-size vector of <see cref="System.Single"/>.</param>
 /// <param name="inputColumnName">Name of the column to take the data from.
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 /// This estimator operates over vector of text.</param>
 public static WordBagEstimator ProduceWordBags(this TransformsCatalog.TextTransforms catalog,
 string outputColumnName,
 char termSeparator,
 char freqSeparator,
 string inputColumnName = null,
 int maximumNgramsCount = NgramExtractingEstimator.Defaults.MaximumNgramsCount)
 => new WordBagEstimator(Contracts.CheckRef(catalog, nameof(catalog)).GetEnvironment(),
 outputColumnName, inputColumnName, 1, 0, true, maximumNgramsCount, NgramExtractingEstimator.WeightingCriteria.Tf, termSeparator: termSeparator, freqSeparator: freqSeparator);

 /// <summary>
 /// Create a <see cref="WordBagEstimator"/>, which maps the multiple columns specified in <paramref name="inputColumnNames"/>
 /// to a vector of n-gram counts in a new column named <paramref name="outputColumnName"/>.
Expand Down

200 changes: 198 additions & 2 deletions
src/Microsoft.ML.Transforms/Text/WordBagTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
Expand All@@ -12,6 +13,7 @@
using Microsoft.ML.Internal.Utilities;
using Microsoft.ML.Runtime;
using Microsoft.ML.Transforms.Text;
using static Microsoft.ML.Transforms.Text.WordBagBuildingTransformer;

[assembly: LoadableClass(WordBagBuildingTransformer.Summary, typeof(IDataTransform), typeof(WordBagBuildingTransformer), typeof(WordBagBuildingTransformer.Options), typeof(SignatureDataTransform),
 "Word Bag Transform", "WordBagTransform", "WordBag")]
Expand All@@ -21,6 +23,16 @@

[assembly: EntryPointModule(typeof(NgramExtractorTransform.NgramExtractorArguments))]

// These are for the internal only TextExpandingTransformer. Not exposed publically
[assembly: LoadableClass(TextExpandingTransformer.Summary, typeof(IDataTransform), typeof(TextExpandingTransformer), null, typeof(SignatureLoadDataTransform),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

[assembly: LoadableClass(typeof(TextExpandingTransformer), null, typeof(SignatureLoadModel),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

[assembly: LoadableClass(typeof(IRowMapper), typeof(TextExpandingTransformer), null, typeof(SignatureLoadRowMapper),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

namespace Microsoft.ML.Transforms.Text
{
 /// <summary>
Expand DownExpand Up@@ -144,18 +156,195 @@ internal static IEstimator<ITransformer> CreateEstimator(IHostEnvironment env, O
 NgramLength = column.NgramLength,
 SkipLength = column.SkipLength,
 Weighting = column.Weighting,
 UseAllLengths = column.UseAllLengths
 UseAllLengths = column.UseAllLengths,
 };
 }

 IEstimator<ITransformer> estimator = NgramExtractionUtils.GetConcatEstimator(h, options.Columns);
 estimator = estimator.Append(new WordTokenizingEstimator(env, tokenizeColumns));
 if (options.FreqSeparator != default)
 {
 estimator = estimator.Append(new TextExpandingEstimator(h, tokenizeColumns[0].InputColumnName, options.FreqSeparator, options.TermSeparator));
 }
 estimator = estimator.Append(new WordTokenizingEstimator(h, tokenizeColumns));
 estimator = estimator.Append(NgramExtractorTransform.CreateEstimator(h, extractorArgs, estimator.GetOutputSchema(inputSchema)));
 return estimator;
 }

 internal static IDataTransform Create(IHostEnvironment env, Options options, IDataView input) =>
 (IDataTransform)CreateEstimator(env, options, SchemaShape.Create(input.Schema)).Fit(input).Transform(input);

 #region TextExpander

 // Internal only estimator used to facilitate the expansion of ngrams with pre-defined weights
 internal sealed class TextExpandingEstimator : TrivialEstimator<TextExpandingTransformer>
 {
 private readonly string _columnName;
 public TextExpandingEstimator(IHostEnvironment env, string columnName, char freqSeparator, char termSeparator)
 : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(TextExpandingEstimator)), new TextExpandingTransformer(env, columnName, freqSeparator, termSeparator))
 {
 _columnName = columnName;
 }

 public override SchemaShape GetOutputSchema(SchemaShape inputSchema)
 {
 Host.CheckValue(inputSchema, nameof(inputSchema));
 if (!inputSchema.TryFindColumn(_columnName, out SchemaShape.Column outCol) && outCol.ItemType != TextDataViewType.Instance)
 {
 throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", _columnName);
 }

 return inputSchema;
 }
 }

 // Internal only transformer used to facilitate the expansion of ngrams with pre-defined weights
 internal sealed class TextExpandingTransformer : RowToRowTransformerBase
 {
 internal const string Summary = "Expands text in the format of term:freq; to have the correct number of terms";
 internal const string UserName = "Text Expanding Transform";
 internal const string LoadName = "TextExpand";

 internal const string LoaderSignature = "TextExpandTransform";

 private readonly string _columnName;
 private readonly char _freqSeparator;
 private readonly char _termSeparator;

 public TextExpandingTransformer(IHostEnvironment env, string columnName, char freqSeparator, char termSeparator)
 : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(TextExpandingTransformer)))
 {
 _columnName = columnName;
 _freqSeparator = freqSeparator;
 _termSeparator = termSeparator;
 }

 private static VersionInfo GetVersionInfo()
 {
 return new VersionInfo(
 modelSignature: "TEXT EXP",
 verWrittenCur: 0x00010001, // Initial
 verReadableCur: 0x00010001,
 verWeCanReadBack: 0x00010001,
 loaderSignature: LoaderSignature,
 loaderAssemblyName: typeof(TextExpandingTransformer).Assembly.FullName);
 }

 /// <summary>
 /// Factory method for SignatureLoadModel.
 /// </summary>
 private TextExpandingTransformer(IHostEnvironment env, ModelLoadContext ctx) :
 base(Contracts.CheckRef(env, nameof(env)).Register(nameof(ColumnConcatenatingTransformer)))
 {
 Host.CheckValue(ctx, nameof(ctx));
 ctx.CheckAtModel(GetVersionInfo());
 // *** Binary format ***
 // string: column n ame
 // char: frequency separator
 // char: term separator

 _columnName = ctx.Reader.ReadString();
 _freqSeparator = ctx.Reader.ReadChar();
 _termSeparator = ctx.Reader.ReadChar();
 }

 /// <summary>
 /// Factory method for SignatureLoadRowMapper.
 /// </summary>
 private static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, DataViewSchema inputSchema)
 => new TextExpandingTransformer(env, ctx).MakeRowMapper(inputSchema);

 /// <summary>
 /// Factory method for SignatureLoadDataTransform.
 /// </summary>
 private static IDataTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
 => new TextExpandingTransformer(env, ctx).MakeDataTransform(input);

 private protected override IRowMapper MakeRowMapper(DataViewSchema schema)
 {
 return new Mapper(Host, schema, this);
 }

 private protected override void SaveModel(ModelSaveContext ctx)
 {
 Host.CheckValue(ctx, nameof(ctx));
 ctx.CheckAtModel();
 ctx.SetVersionInfo(GetVersionInfo());

 // *** Binary format ***
 // string: column n ame
 // char: frequency separator
 // char: term separator

 ctx.Writer.Write(_columnName);
 ctx.Writer.Write(_freqSeparator);
 ctx.Writer.Write(_termSeparator);
 }

 private sealed class Mapper : MapperBase
 {
 private readonly TextExpandingTransformer _parent;
 public Mapper(IHost host, DataViewSchema inputSchema, RowToRowTransformerBase parent)
 : base(host, inputSchema, parent)
 {
 _parent = (TextExpandingTransformer)parent;
 }

 protected override DataViewSchema.DetachedColumn[] GetOutputColumnsCore()
 {
 return new DataViewSchema.DetachedColumn[]
 {
 new DataViewSchema.DetachedColumn(_parent._columnName, TextDataViewType.Instance)
 };
 }

 protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func<int, bool> activeOutput, out Action disposer)
 {
 disposer = null;
 ValueGetter<ReadOnlyMemory<char>> srcGetter = input.GetGetter<ReadOnlyMemory<char>>(input.Schema.GetColumnOrNull(_parent._columnName).Value);
 ReadOnlyMemory<char> inputMem = default;
 var sb = new StringBuilder();

 ValueGetter<ReadOnlyMemory<char>> result = (ref ReadOnlyMemory<char> dst) =>
 {
 sb.Clear();
 srcGetter(ref inputMem);
 var inputText = inputMem.ToString();
 foreach (var termFreq in inputText.Split(_parent._termSeparator))
 {
 var tf = termFreq.Split(_parent._freqSeparator);
 if (tf.Length != 2)
 sb.Append(tf[0] + " ");
 else
 {
 for (int i = 0; i < int.Parse(tf[1]); i++)
 sb.Append(tf[0] + " ");
 }
 }

 dst = sb.ToString().AsMemory();
 };

 return result;
 }

 private protected override Func<int, bool> GetDependenciesCore(Func<int, bool> activeOutput)
 {
 var active = new bool[InputSchema.Count];
 if (activeOutput(0))
 {
 active[InputSchema.GetColumnOrNull(_parent._columnName).Value.Index] = true;
 }
 return col => active[col];
 }

 private protected override void SaveModel(ModelSaveContext ctx)
 {
 _parent.SaveModel(ctx);
 }
 }
 }

 #endregion TextExpander
 }

 /// <summary>
Expand DownExpand Up@@ -235,6 +424,13 @@ internal abstract class ArgumentsBase

 [Argument(ArgumentType.AtMostOnce, HelpText = "The weighting criteria")]
 public NgramExtractingEstimator.WeightingCriteria Weighting = NgramExtractingEstimator.Defaults.Weighting;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Separator used to separate terms/frequency pairs.")]
 public char TermSeparator = default;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Separator used to separate terms from their frequency.")]
 public char FreqSeparator = default;

 }

 [TlcModule.Component(Name = "NGram", FriendlyName = "NGram Extractor Transform", Alias = "NGramExtractorTransform,NGramExtractor",
Expand Down

30 changes: 24 additions & 6 deletions
src/Microsoft.ML.Transforms/Text/WrappedTextTransformers.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,6 +45,8 @@ public sealed class WordBagEstimator : IEstimator<ITransformer>
 private readonly bool _useAllLengths;
 private readonly int _maxNumTerms;
 private readonly NgramExtractingEstimator.WeightingCriteria _weighting;
 private readonly char _termSeparator;
 private readonly char _freqSeparator;

 /// <summary>
 /// Options for how the n-grams are extracted.
Expand DownExpand Up@@ -99,15 +101,19 @@ public Options()
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 string outputColumnName,
 string inputColumnName = null,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 : this(env, outputColumnName, new[] { inputColumnName ?? outputColumnName }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 : this(env, outputColumnName, new[] { inputColumnName ?? outputColumnName }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting, termSeparator, freqSeparator)
 {
 }

Expand All@@ -123,15 +129,19 @@ internal WordBagEstimator(IHostEnvironment env,
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 string outputColumnName,
 string[] inputColumnNames,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 : this(env, new[] { (outputColumnName, inputColumnNames) }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 : this(env, new[] { (outputColumnName, inputColumnNames) }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting, termSeparator, freqSeparator)
 {
 }

Expand All@@ -146,13 +156,17 @@ internal WordBagEstimator(IHostEnvironment env,
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 (string outputColumnName, string[] inputColumnNames)[] columns,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 {
 Contracts.CheckValue(env, nameof(env));
 _host = env.Register(nameof(WordBagEstimator));
Expand All@@ -169,6 +183,8 @@ internal WordBagEstimator(IHostEnvironment env,
 _useAllLengths = useAllLengths;
 _maxNumTerms = maximumNgramsCount;
 _weighting = weighting;
 _termSeparator = termSeparator;
 _freqSeparator = freqSeparator;
 }

 /// <summary> Trains and returns a <see cref="ITransformer"/>.</summary>
Expand All@@ -187,7 +203,9 @@ private WordBagBuildingTransformer.Options CreateOptions()
 SkipLength = _skipLength,
 UseAllLengths = _useAllLengths,
 MaxNumTerms = new[] { _maxNumTerms },
 Weighting = _weighting
 Weighting = _weighting,
 TermSeparator = _termSeparator,
 FreqSeparator = _freqSeparator,
 };
 }

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Add in ability to have pre-defined weights for ngrams.#6458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

michaelgsharp
merged 5 commits into

from

Nov 23, 2022
 Merged


Add in ability to have pre-defined weights for ngrams.
#6458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
b91fe6f

ngram pre defined weights


michaelgsharp Nov 17, 2022
0bed831

removed commented code


michaelgsharp Nov 17, 2022
a436e61

formatting fix


michaelgsharp Nov 17, 2022
88d7557

renegerate entry points


michaelgsharp Nov 17, 2022
61044f5

fixing API


michaelgsharp Nov 23, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


25 changes: 25 additions & 0 deletions
src/Microsoft.ML.Transforms/Text/TextCatalog.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -362,6 +362,31 @@ public static WordBagEstimator ProduceWordBags(this TransformsCatalog.TextTransf
 => new WordBagEstimator(Contracts.CheckRef(catalog, nameof(catalog)).GetEnvironment(),
 outputColumnName, inputColumnName, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting);

 /// <summary>
 /// Create a <see cref="WordBagEstimator"/>, which maps the column specified in <paramref name="inputColumnName"/>
 /// to a vector of n-gram counts in a new column named <paramref name="outputColumnName"/>.
 /// </summary>
 /// <remarks>
 /// <see cref="WordBagEstimator"/> is different from <see cref="NgramExtractingEstimator"/> in that the former
 /// tokenizes text internally and the latter takes tokenized text as input.
 /// </remarks>
 /// <param name="catalog">The transform's catalog.</param>
 /// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.
 /// This column's data type will be known-size vector of <see cref="System.Single"/>.</param>
 /// <param name="inputColumnName">Name of the column to take the data from.
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 /// This estimator operates over vector of text.</param>
 public static WordBagEstimator ProduceWordBags(this TransformsCatalog.TextTransforms catalog,
 string outputColumnName,
 char termSeparator,
 char freqSeparator,
 string inputColumnName = null,
 int maximumNgramsCount = NgramExtractingEstimator.Defaults.MaximumNgramsCount)
 => new WordBagEstimator(Contracts.CheckRef(catalog, nameof(catalog)).GetEnvironment(),
 outputColumnName, inputColumnName, 1, 0, true, maximumNgramsCount, NgramExtractingEstimator.WeightingCriteria.Tf, termSeparator: termSeparator, freqSeparator: freqSeparator);

 /// <summary>
 /// Create a <see cref="WordBagEstimator"/>, which maps the multiple columns specified in <paramref name="inputColumnNames"/>
 /// to a vector of n-gram counts in a new column named <paramref name="outputColumnName"/>.
Expand Down

200 changes: 198 additions & 2 deletions
src/Microsoft.ML.Transforms/Text/WordBagTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
Expand All@@ -12,6 +13,7 @@
using Microsoft.ML.Internal.Utilities;
using Microsoft.ML.Runtime;
using Microsoft.ML.Transforms.Text;
using static Microsoft.ML.Transforms.Text.WordBagBuildingTransformer;

[assembly: LoadableClass(WordBagBuildingTransformer.Summary, typeof(IDataTransform), typeof(WordBagBuildingTransformer), typeof(WordBagBuildingTransformer.Options), typeof(SignatureDataTransform),
 "Word Bag Transform", "WordBagTransform", "WordBag")]
Expand All@@ -21,6 +23,16 @@

[assembly: EntryPointModule(typeof(NgramExtractorTransform.NgramExtractorArguments))]

// These are for the internal only TextExpandingTransformer. Not exposed publically
[assembly: LoadableClass(TextExpandingTransformer.Summary, typeof(IDataTransform), typeof(TextExpandingTransformer), null, typeof(SignatureLoadDataTransform),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

[assembly: LoadableClass(typeof(TextExpandingTransformer), null, typeof(SignatureLoadModel),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

[assembly: LoadableClass(typeof(IRowMapper), typeof(TextExpandingTransformer), null, typeof(SignatureLoadRowMapper),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

namespace Microsoft.ML.Transforms.Text
{
 /// <summary>
Expand DownExpand Up@@ -144,18 +156,195 @@ internal static IEstimator<ITransformer> CreateEstimator(IHostEnvironment env, O
 NgramLength = column.NgramLength,
 SkipLength = column.SkipLength,
 Weighting = column.Weighting,
 UseAllLengths = column.UseAllLengths
 UseAllLengths = column.UseAllLengths,
 };
 }

 IEstimator<ITransformer> estimator = NgramExtractionUtils.GetConcatEstimator(h, options.Columns);
 estimator = estimator.Append(new WordTokenizingEstimator(env, tokenizeColumns));
 if (options.FreqSeparator != default)
 {
 estimator = estimator.Append(new TextExpandingEstimator(h, tokenizeColumns[0].InputColumnName, options.FreqSeparator, options.TermSeparator));
 }
 estimator = estimator.Append(new WordTokenizingEstimator(h, tokenizeColumns));
 estimator = estimator.Append(NgramExtractorTransform.CreateEstimator(h, extractorArgs, estimator.GetOutputSchema(inputSchema)));
 return estimator;
 }

 internal static IDataTransform Create(IHostEnvironment env, Options options, IDataView input) =>
 (IDataTransform)CreateEstimator(env, options, SchemaShape.Create(input.Schema)).Fit(input).Transform(input);

 #region TextExpander

 // Internal only estimator used to facilitate the expansion of ngrams with pre-defined weights
 internal sealed class TextExpandingEstimator : TrivialEstimator<TextExpandingTransformer>
 {
 private readonly string _columnName;
 public TextExpandingEstimator(IHostEnvironment env, string columnName, char freqSeparator, char termSeparator)
 : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(TextExpandingEstimator)), new TextExpandingTransformer(env, columnName, freqSeparator, termSeparator))
 {
 _columnName = columnName;
 }

 public override SchemaShape GetOutputSchema(SchemaShape inputSchema)
 {
 Host.CheckValue(inputSchema, nameof(inputSchema));
 if (!inputSchema.TryFindColumn(_columnName, out SchemaShape.Column outCol) && outCol.ItemType != TextDataViewType.Instance)
 {
 throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", _columnName);
 }

 return inputSchema;
 }
 }

 // Internal only transformer used to facilitate the expansion of ngrams with pre-defined weights
 internal sealed class TextExpandingTransformer : RowToRowTransformerBase
 {
 internal const string Summary = "Expands text in the format of term:freq; to have the correct number of terms";
 internal const string UserName = "Text Expanding Transform";
 internal const string LoadName = "TextExpand";

 internal const string LoaderSignature = "TextExpandTransform";

 private readonly string _columnName;
 private readonly char _freqSeparator;
 private readonly char _termSeparator;

 public TextExpandingTransformer(IHostEnvironment env, string columnName, char freqSeparator, char termSeparator)
 : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(TextExpandingTransformer)))
 {
 _columnName = columnName;
 _freqSeparator = freqSeparator;
 _termSeparator = termSeparator;
 }

 private static VersionInfo GetVersionInfo()
 {
 return new VersionInfo(
 modelSignature: "TEXT EXP",
 verWrittenCur: 0x00010001, // Initial
 verReadableCur: 0x00010001,
 verWeCanReadBack: 0x00010001,
 loaderSignature: LoaderSignature,
 loaderAssemblyName: typeof(TextExpandingTransformer).Assembly.FullName);
 }

 /// <summary>
 /// Factory method for SignatureLoadModel.
 /// </summary>
 private TextExpandingTransformer(IHostEnvironment env, ModelLoadContext ctx) :
 base(Contracts.CheckRef(env, nameof(env)).Register(nameof(ColumnConcatenatingTransformer)))
 {
 Host.CheckValue(ctx, nameof(ctx));
 ctx.CheckAtModel(GetVersionInfo());
 // *** Binary format ***
 // string: column n ame
 // char: frequency separator
 // char: term separator

 _columnName = ctx.Reader.ReadString();
 _freqSeparator = ctx.Reader.ReadChar();
 _termSeparator = ctx.Reader.ReadChar();
 }

 /// <summary>
 /// Factory method for SignatureLoadRowMapper.
 /// </summary>
 private static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, DataViewSchema inputSchema)
 => new TextExpandingTransformer(env, ctx).MakeRowMapper(inputSchema);

 /// <summary>
 /// Factory method for SignatureLoadDataTransform.
 /// </summary>
 private static IDataTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
 => new TextExpandingTransformer(env, ctx).MakeDataTransform(input);

 private protected override IRowMapper MakeRowMapper(DataViewSchema schema)
 {
 return new Mapper(Host, schema, this);
 }

 private protected override void SaveModel(ModelSaveContext ctx)
 {
 Host.CheckValue(ctx, nameof(ctx));
 ctx.CheckAtModel();
 ctx.SetVersionInfo(GetVersionInfo());

 // *** Binary format ***
 // string: column n ame
 // char: frequency separator
 // char: term separator

 ctx.Writer.Write(_columnName);
 ctx.Writer.Write(_freqSeparator);
 ctx.Writer.Write(_termSeparator);
 }

 private sealed class Mapper : MapperBase
 {
 private readonly TextExpandingTransformer _parent;
 public Mapper(IHost host, DataViewSchema inputSchema, RowToRowTransformerBase parent)
 : base(host, inputSchema, parent)
 {
 _parent = (TextExpandingTransformer)parent;
 }

 protected override DataViewSchema.DetachedColumn[] GetOutputColumnsCore()
 {
 return new DataViewSchema.DetachedColumn[]
 {
 new DataViewSchema.DetachedColumn(_parent._columnName, TextDataViewType.Instance)
 };
 }

 protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func<int, bool> activeOutput, out Action disposer)
 {
 disposer = null;
 ValueGetter<ReadOnlyMemory<char>> srcGetter = input.GetGetter<ReadOnlyMemory<char>>(input.Schema.GetColumnOrNull(_parent._columnName).Value);
 ReadOnlyMemory<char> inputMem = default;
 var sb = new StringBuilder();

 ValueGetter<ReadOnlyMemory<char>> result = (ref ReadOnlyMemory<char> dst) =>
 {
 sb.Clear();
 srcGetter(ref inputMem);
 var inputText = inputMem.ToString();
 foreach (var termFreq in inputText.Split(_parent._termSeparator))
 {
 var tf = termFreq.Split(_parent._freqSeparator);
 if (tf.Length != 2)
 sb.Append(tf[0] + " ");
 else
 {
 for (int i = 0; i < int.Parse(tf[1]); i++)
 sb.Append(tf[0] + " ");
 }
 }

 dst = sb.ToString().AsMemory();
 };

 return result;
 }

 private protected override Func<int, bool> GetDependenciesCore(Func<int, bool> activeOutput)
 {
 var active = new bool[InputSchema.Count];
 if (activeOutput(0))
 {
 active[InputSchema.GetColumnOrNull(_parent._columnName).Value.Index] = true;
 }
 return col => active[col];
 }

 private protected override void SaveModel(ModelSaveContext ctx)
 {
 _parent.SaveModel(ctx);
 }
 }
 }

 #endregion TextExpander
 }

 /// <summary>
Expand DownExpand Up@@ -235,6 +424,13 @@ internal abstract class ArgumentsBase

 [Argument(ArgumentType.AtMostOnce, HelpText = "The weighting criteria")]
 public NgramExtractingEstimator.WeightingCriteria Weighting = NgramExtractingEstimator.Defaults.Weighting;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Separator used to separate terms/frequency pairs.")]
 public char TermSeparator = default;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Separator used to separate terms from their frequency.")]
 public char FreqSeparator = default;

 }

 [TlcModule.Component(Name = "NGram", FriendlyName = "NGram Extractor Transform", Alias = "NGramExtractorTransform,NGramExtractor",
Expand Down

30 changes: 24 additions & 6 deletions
src/Microsoft.ML.Transforms/Text/WrappedTextTransformers.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,6 +45,8 @@ public sealed class WordBagEstimator : IEstimator<ITransformer>
 private readonly bool _useAllLengths;
 private readonly int _maxNumTerms;
 private readonly NgramExtractingEstimator.WeightingCriteria _weighting;
 private readonly char _termSeparator;
 private readonly char _freqSeparator;

 /// <summary>
 /// Options for how the n-grams are extracted.
Expand DownExpand Up@@ -99,15 +101,19 @@ public Options()
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 string outputColumnName,
 string inputColumnName = null,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 : this(env, outputColumnName, new[] { inputColumnName ?? outputColumnName }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 : this(env, outputColumnName, new[] { inputColumnName ?? outputColumnName }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting, termSeparator, freqSeparator)
 {
 }

Expand All@@ -123,15 +129,19 @@ internal WordBagEstimator(IHostEnvironment env,
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 string outputColumnName,
 string[] inputColumnNames,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 : this(env, new[] { (outputColumnName, inputColumnNames) }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 : this(env, new[] { (outputColumnName, inputColumnNames) }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting, termSeparator, freqSeparator)
 {
 }

Expand All@@ -146,13 +156,17 @@ internal WordBagEstimator(IHostEnvironment env,
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 (string outputColumnName, string[] inputColumnNames)[] columns,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 {
 Contracts.CheckValue(env, nameof(env));
 _host = env.Register(nameof(WordBagEstimator));
Expand All@@ -169,6 +183,8 @@ internal WordBagEstimator(IHostEnvironment env,
 _useAllLengths = useAllLengths;
 _maxNumTerms = maximumNgramsCount;
 _weighting = weighting;
 _termSeparator = termSeparator;
 _freqSeparator = freqSeparator;
 }

 /// <summary> Trains and returns a <see cref="ITransformer"/>.</summary>
Expand All@@ -187,7 +203,9 @@ private WordBagBuildingTransformer.Options CreateOptions()
 SkipLength = _skipLength,
 UseAllLengths = _useAllLengths,
 MaxNumTerms = new[] { _maxNumTerms },
 Weighting = _weighting
 Weighting = _weighting,
 TermSeparator = _termSeparator,
 FreqSeparator = _freqSeparator,
 };
 }

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Add in ability to have pre-defined weights for ngrams.#6458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

michaelgsharp
merged 5 commits into

from

Nov 23, 2022
 Merged


Add in ability to have pre-defined weights for ngrams.
#6458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
b91fe6f

ngram pre defined weights


michaelgsharp Nov 17, 2022
0bed831

removed commented code


michaelgsharp Nov 17, 2022
a436e61

formatting fix


michaelgsharp Nov 17, 2022
88d7557

renegerate entry points


michaelgsharp Nov 17, 2022
61044f5

fixing API


michaelgsharp Nov 23, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


25 changes: 25 additions & 0 deletions
src/Microsoft.ML.Transforms/Text/TextCatalog.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -362,6 +362,31 @@ public static WordBagEstimator ProduceWordBags(this TransformsCatalog.TextTransf
 => new WordBagEstimator(Contracts.CheckRef(catalog, nameof(catalog)).GetEnvironment(),
 outputColumnName, inputColumnName, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting);

 /// <summary>
 /// Create a <see cref="WordBagEstimator"/>, which maps the column specified in <paramref name="inputColumnName"/>
 /// to a vector of n-gram counts in a new column named <paramref name="outputColumnName"/>.
 /// </summary>
 /// <remarks>
 /// <see cref="WordBagEstimator"/> is different from <see cref="NgramExtractingEstimator"/> in that the former
 /// tokenizes text internally and the latter takes tokenized text as input.
 /// </remarks>
 /// <param name="catalog">The transform's catalog.</param>
 /// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.
 /// This column's data type will be known-size vector of <see cref="System.Single"/>.</param>
 /// <param name="inputColumnName">Name of the column to take the data from.
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 /// This estimator operates over vector of text.</param>
 public static WordBagEstimator ProduceWordBags(this TransformsCatalog.TextTransforms catalog,
 string outputColumnName,
 char termSeparator,
 char freqSeparator,
 string inputColumnName = null,
 int maximumNgramsCount = NgramExtractingEstimator.Defaults.MaximumNgramsCount)
 => new WordBagEstimator(Contracts.CheckRef(catalog, nameof(catalog)).GetEnvironment(),
 outputColumnName, inputColumnName, 1, 0, true, maximumNgramsCount, NgramExtractingEstimator.WeightingCriteria.Tf, termSeparator: termSeparator, freqSeparator: freqSeparator);

 /// <summary>
 /// Create a <see cref="WordBagEstimator"/>, which maps the multiple columns specified in <paramref name="inputColumnNames"/>
 /// to a vector of n-gram counts in a new column named <paramref name="outputColumnName"/>.
Expand Down

200 changes: 198 additions & 2 deletions
src/Microsoft.ML.Transforms/Text/WordBagTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
Expand All@@ -12,6 +13,7 @@
using Microsoft.ML.Internal.Utilities;
using Microsoft.ML.Runtime;
using Microsoft.ML.Transforms.Text;
using static Microsoft.ML.Transforms.Text.WordBagBuildingTransformer;

[assembly: LoadableClass(WordBagBuildingTransformer.Summary, typeof(IDataTransform), typeof(WordBagBuildingTransformer), typeof(WordBagBuildingTransformer.Options), typeof(SignatureDataTransform),
 "Word Bag Transform", "WordBagTransform", "WordBag")]
Expand All@@ -21,6 +23,16 @@

[assembly: EntryPointModule(typeof(NgramExtractorTransform.NgramExtractorArguments))]

// These are for the internal only TextExpandingTransformer. Not exposed publically
[assembly: LoadableClass(TextExpandingTransformer.Summary, typeof(IDataTransform), typeof(TextExpandingTransformer), null, typeof(SignatureLoadDataTransform),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

[assembly: LoadableClass(typeof(TextExpandingTransformer), null, typeof(SignatureLoadModel),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

[assembly: LoadableClass(typeof(IRowMapper), typeof(TextExpandingTransformer), null, typeof(SignatureLoadRowMapper),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

namespace Microsoft.ML.Transforms.Text
{
 /// <summary>
Expand DownExpand Up@@ -144,18 +156,195 @@ internal static IEstimator<ITransformer> CreateEstimator(IHostEnvironment env, O
 NgramLength = column.NgramLength,
 SkipLength = column.SkipLength,
 Weighting = column.Weighting,
 UseAllLengths = column.UseAllLengths
 UseAllLengths = column.UseAllLengths,
 };
 }

 IEstimator<ITransformer> estimator = NgramExtractionUtils.GetConcatEstimator(h, options.Columns);
 estimator = estimator.Append(new WordTokenizingEstimator(env, tokenizeColumns));
 if (options.FreqSeparator != default)
 {
 estimator = estimator.Append(new TextExpandingEstimator(h, tokenizeColumns[0].InputColumnName, options.FreqSeparator, options.TermSeparator));
 }
 estimator = estimator.Append(new WordTokenizingEstimator(h, tokenizeColumns));
 estimator = estimator.Append(NgramExtractorTransform.CreateEstimator(h, extractorArgs, estimator.GetOutputSchema(inputSchema)));
 return estimator;
 }

 internal static IDataTransform Create(IHostEnvironment env, Options options, IDataView input) =>
 (IDataTransform)CreateEstimator(env, options, SchemaShape.Create(input.Schema)).Fit(input).Transform(input);

 #region TextExpander

 // Internal only estimator used to facilitate the expansion of ngrams with pre-defined weights
 internal sealed class TextExpandingEstimator : TrivialEstimator<TextExpandingTransformer>
 {
 private readonly string _columnName;
 public TextExpandingEstimator(IHostEnvironment env, string columnName, char freqSeparator, char termSeparator)
 : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(TextExpandingEstimator)), new TextExpandingTransformer(env, columnName, freqSeparator, termSeparator))
 {
 _columnName = columnName;
 }

 public override SchemaShape GetOutputSchema(SchemaShape inputSchema)
 {
 Host.CheckValue(inputSchema, nameof(inputSchema));
 if (!inputSchema.TryFindColumn(_columnName, out SchemaShape.Column outCol) && outCol.ItemType != TextDataViewType.Instance)
 {
 throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", _columnName);
 }

 return inputSchema;
 }
 }

 // Internal only transformer used to facilitate the expansion of ngrams with pre-defined weights
 internal sealed class TextExpandingTransformer : RowToRowTransformerBase
 {
 internal const string Summary = "Expands text in the format of term:freq; to have the correct number of terms";
 internal const string UserName = "Text Expanding Transform";
 internal const string LoadName = "TextExpand";

 internal const string LoaderSignature = "TextExpandTransform";

 private readonly string _columnName;
 private readonly char _freqSeparator;
 private readonly char _termSeparator;

 public TextExpandingTransformer(IHostEnvironment env, string columnName, char freqSeparator, char termSeparator)
 : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(TextExpandingTransformer)))
 {
 _columnName = columnName;
 _freqSeparator = freqSeparator;
 _termSeparator = termSeparator;
 }

 private static VersionInfo GetVersionInfo()
 {
 return new VersionInfo(
 modelSignature: "TEXT EXP",
 verWrittenCur: 0x00010001, // Initial
 verReadableCur: 0x00010001,
 verWeCanReadBack: 0x00010001,
 loaderSignature: LoaderSignature,
 loaderAssemblyName: typeof(TextExpandingTransformer).Assembly.FullName);
 }

 /// <summary>
 /// Factory method for SignatureLoadModel.
 /// </summary>
 private TextExpandingTransformer(IHostEnvironment env, ModelLoadContext ctx) :
 base(Contracts.CheckRef(env, nameof(env)).Register(nameof(ColumnConcatenatingTransformer)))
 {
 Host.CheckValue(ctx, nameof(ctx));
 ctx.CheckAtModel(GetVersionInfo());
 // *** Binary format ***
 // string: column n ame
 // char: frequency separator
 // char: term separator

 _columnName = ctx.Reader.ReadString();
 _freqSeparator = ctx.Reader.ReadChar();
 _termSeparator = ctx.Reader.ReadChar();
 }

 /// <summary>
 /// Factory method for SignatureLoadRowMapper.
 /// </summary>
 private static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, DataViewSchema inputSchema)
 => new TextExpandingTransformer(env, ctx).MakeRowMapper(inputSchema);

 /// <summary>
 /// Factory method for SignatureLoadDataTransform.
 /// </summary>
 private static IDataTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
 => new TextExpandingTransformer(env, ctx).MakeDataTransform(input);

 private protected override IRowMapper MakeRowMapper(DataViewSchema schema)
 {
 return new Mapper(Host, schema, this);
 }

 private protected override void SaveModel(ModelSaveContext ctx)
 {
 Host.CheckValue(ctx, nameof(ctx));
 ctx.CheckAtModel();
 ctx.SetVersionInfo(GetVersionInfo());

 // *** Binary format ***
 // string: column n ame
 // char: frequency separator
 // char: term separator

 ctx.Writer.Write(_columnName);
 ctx.Writer.Write(_freqSeparator);
 ctx.Writer.Write(_termSeparator);
 }

 private sealed class Mapper : MapperBase
 {
 private readonly TextExpandingTransformer _parent;
 public Mapper(IHost host, DataViewSchema inputSchema, RowToRowTransformerBase parent)
 : base(host, inputSchema, parent)
 {
 _parent = (TextExpandingTransformer)parent;
 }

 protected override DataViewSchema.DetachedColumn[] GetOutputColumnsCore()
 {
 return new DataViewSchema.DetachedColumn[]
 {
 new DataViewSchema.DetachedColumn(_parent._columnName, TextDataViewType.Instance)
 };
 }

 protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func<int, bool> activeOutput, out Action disposer)
 {
 disposer = null;
 ValueGetter<ReadOnlyMemory<char>> srcGetter = input.GetGetter<ReadOnlyMemory<char>>(input.Schema.GetColumnOrNull(_parent._columnName).Value);
 ReadOnlyMemory<char> inputMem = default;
 var sb = new StringBuilder();

 ValueGetter<ReadOnlyMemory<char>> result = (ref ReadOnlyMemory<char> dst) =>
 {
 sb.Clear();
 srcGetter(ref inputMem);
 var inputText = inputMem.ToString();
 foreach (var termFreq in inputText.Split(_parent._termSeparator))
 {
 var tf = termFreq.Split(_parent._freqSeparator);
 if (tf.Length != 2)
 sb.Append(tf[0] + " ");
 else
 {
 for (int i = 0; i < int.Parse(tf[1]); i++)
 sb.Append(tf[0] + " ");
 }
 }

 dst = sb.ToString().AsMemory();
 };

 return result;
 }

 private protected override Func<int, bool> GetDependenciesCore(Func<int, bool> activeOutput)
 {
 var active = new bool[InputSchema.Count];
 if (activeOutput(0))
 {
 active[InputSchema.GetColumnOrNull(_parent._columnName).Value.Index] = true;
 }
 return col => active[col];
 }

 private protected override void SaveModel(ModelSaveContext ctx)
 {
 _parent.SaveModel(ctx);
 }
 }
 }

 #endregion TextExpander
 }

 /// <summary>
Expand DownExpand Up@@ -235,6 +424,13 @@ internal abstract class ArgumentsBase

 [Argument(ArgumentType.AtMostOnce, HelpText = "The weighting criteria")]
 public NgramExtractingEstimator.WeightingCriteria Weighting = NgramExtractingEstimator.Defaults.Weighting;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Separator used to separate terms/frequency pairs.")]
 public char TermSeparator = default;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Separator used to separate terms from their frequency.")]
 public char FreqSeparator = default;

 }

 [TlcModule.Component(Name = "NGram", FriendlyName = "NGram Extractor Transform", Alias = "NGramExtractorTransform,NGramExtractor",
Expand Down

30 changes: 24 additions & 6 deletions
src/Microsoft.ML.Transforms/Text/WrappedTextTransformers.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,6 +45,8 @@ public sealed class WordBagEstimator : IEstimator<ITransformer>
 private readonly bool _useAllLengths;
 private readonly int _maxNumTerms;
 private readonly NgramExtractingEstimator.WeightingCriteria _weighting;
 private readonly char _termSeparator;
 private readonly char _freqSeparator;

 /// <summary>
 /// Options for how the n-grams are extracted.
Expand DownExpand Up@@ -99,15 +101,19 @@ public Options()
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 string outputColumnName,
 string inputColumnName = null,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 : this(env, outputColumnName, new[] { inputColumnName ?? outputColumnName }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 : this(env, outputColumnName, new[] { inputColumnName ?? outputColumnName }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting, termSeparator, freqSeparator)
 {
 }

Expand All@@ -123,15 +129,19 @@ internal WordBagEstimator(IHostEnvironment env,
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 string outputColumnName,
 string[] inputColumnNames,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 : this(env, new[] { (outputColumnName, inputColumnNames) }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 : this(env, new[] { (outputColumnName, inputColumnNames) }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting, termSeparator, freqSeparator)
 {
 }

Expand All@@ -146,13 +156,17 @@ internal WordBagEstimator(IHostEnvironment env,
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 (string outputColumnName, string[] inputColumnNames)[] columns,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 {
 Contracts.CheckValue(env, nameof(env));
 _host = env.Register(nameof(WordBagEstimator));
Expand All@@ -169,6 +183,8 @@ internal WordBagEstimator(IHostEnvironment env,
 _useAllLengths = useAllLengths;
 _maxNumTerms = maximumNgramsCount;
 _weighting = weighting;
 _termSeparator = termSeparator;
 _freqSeparator = freqSeparator;
 }

 /// <summary> Trains and returns a <see cref="ITransformer"/>.</summary>
Expand All@@ -187,7 +203,9 @@ private WordBagBuildingTransformer.Options CreateOptions()
 SkipLength = _skipLength,
 UseAllLengths = _useAllLengths,
 MaxNumTerms = new[] { _maxNumTerms },
 Weighting = _weighting
 Weighting = _weighting,
 TermSeparator = _termSeparator,
 FreqSeparator = _freqSeparator,
 };
 }

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Add in ability to have pre-defined weights for ngrams.#6458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

michaelgsharp
merged 5 commits into

from

Nov 23, 2022
 Merged


Add in ability to have pre-defined weights for ngrams.
#6458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
b91fe6f

ngram pre defined weights


michaelgsharp Nov 17, 2022
0bed831

removed commented code


michaelgsharp Nov 17, 2022
a436e61

formatting fix


michaelgsharp Nov 17, 2022
88d7557

renegerate entry points


michaelgsharp Nov 17, 2022
61044f5

fixing API


michaelgsharp Nov 23, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


25 changes: 25 additions & 0 deletions
src/Microsoft.ML.Transforms/Text/TextCatalog.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -362,6 +362,31 @@ public static WordBagEstimator ProduceWordBags(this TransformsCatalog.TextTransf
 => new WordBagEstimator(Contracts.CheckRef(catalog, nameof(catalog)).GetEnvironment(),
 outputColumnName, inputColumnName, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting);

 /// <summary>
 /// Create a <see cref="WordBagEstimator"/>, which maps the column specified in <paramref name="inputColumnName"/>
 /// to a vector of n-gram counts in a new column named <paramref name="outputColumnName"/>.
 /// </summary>
 /// <remarks>
 /// <see cref="WordBagEstimator"/> is different from <see cref="NgramExtractingEstimator"/> in that the former
 /// tokenizes text internally and the latter takes tokenized text as input.
 /// </remarks>
 /// <param name="catalog">The transform's catalog.</param>
 /// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.
 /// This column's data type will be known-size vector of <see cref="System.Single"/>.</param>
 /// <param name="inputColumnName">Name of the column to take the data from.
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 /// This estimator operates over vector of text.</param>
 public static WordBagEstimator ProduceWordBags(this TransformsCatalog.TextTransforms catalog,
 string outputColumnName,
 char termSeparator,
 char freqSeparator,
 string inputColumnName = null,
 int maximumNgramsCount = NgramExtractingEstimator.Defaults.MaximumNgramsCount)
 => new WordBagEstimator(Contracts.CheckRef(catalog, nameof(catalog)).GetEnvironment(),
 outputColumnName, inputColumnName, 1, 0, true, maximumNgramsCount, NgramExtractingEstimator.WeightingCriteria.Tf, termSeparator: termSeparator, freqSeparator: freqSeparator);

 /// <summary>
 /// Create a <see cref="WordBagEstimator"/>, which maps the multiple columns specified in <paramref name="inputColumnNames"/>
 /// to a vector of n-gram counts in a new column named <paramref name="outputColumnName"/>.
Expand Down

200 changes: 198 additions & 2 deletions
src/Microsoft.ML.Transforms/Text/WordBagTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
Expand All@@ -12,6 +13,7 @@
using Microsoft.ML.Internal.Utilities;
using Microsoft.ML.Runtime;
using Microsoft.ML.Transforms.Text;
using static Microsoft.ML.Transforms.Text.WordBagBuildingTransformer;

[assembly: LoadableClass(WordBagBuildingTransformer.Summary, typeof(IDataTransform), typeof(WordBagBuildingTransformer), typeof(WordBagBuildingTransformer.Options), typeof(SignatureDataTransform),
 "Word Bag Transform", "WordBagTransform", "WordBag")]
Expand All@@ -21,6 +23,16 @@

[assembly: EntryPointModule(typeof(NgramExtractorTransform.NgramExtractorArguments))]

// These are for the internal only TextExpandingTransformer. Not exposed publically
[assembly: LoadableClass(TextExpandingTransformer.Summary, typeof(IDataTransform), typeof(TextExpandingTransformer), null, typeof(SignatureLoadDataTransform),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

[assembly: LoadableClass(typeof(TextExpandingTransformer), null, typeof(SignatureLoadModel),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

[assembly: LoadableClass(typeof(IRowMapper), typeof(TextExpandingTransformer), null, typeof(SignatureLoadRowMapper),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

namespace Microsoft.ML.Transforms.Text
{
 /// <summary>
Expand DownExpand Up@@ -144,18 +156,195 @@ internal static IEstimator<ITransformer> CreateEstimator(IHostEnvironment env, O
 NgramLength = column.NgramLength,
 SkipLength = column.SkipLength,
 Weighting = column.Weighting,
 UseAllLengths = column.UseAllLengths
 UseAllLengths = column.UseAllLengths,
 };
 }

 IEstimator<ITransformer> estimator = NgramExtractionUtils.GetConcatEstimator(h, options.Columns);
 estimator = estimator.Append(new WordTokenizingEstimator(env, tokenizeColumns));
 if (options.FreqSeparator != default)
 {
 estimator = estimator.Append(new TextExpandingEstimator(h, tokenizeColumns[0].InputColumnName, options.FreqSeparator, options.TermSeparator));
 }
 estimator = estimator.Append(new WordTokenizingEstimator(h, tokenizeColumns));
 estimator = estimator.Append(NgramExtractorTransform.CreateEstimator(h, extractorArgs, estimator.GetOutputSchema(inputSchema)));
 return estimator;
 }

 internal static IDataTransform Create(IHostEnvironment env, Options options, IDataView input) =>
 (IDataTransform)CreateEstimator(env, options, SchemaShape.Create(input.Schema)).Fit(input).Transform(input);

 #region TextExpander

 // Internal only estimator used to facilitate the expansion of ngrams with pre-defined weights
 internal sealed class TextExpandingEstimator : TrivialEstimator<TextExpandingTransformer>
 {
 private readonly string _columnName;
 public TextExpandingEstimator(IHostEnvironment env, string columnName, char freqSeparator, char termSeparator)
 : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(TextExpandingEstimator)), new TextExpandingTransformer(env, columnName, freqSeparator, termSeparator))
 {
 _columnName = columnName;
 }

 public override SchemaShape GetOutputSchema(SchemaShape inputSchema)
 {
 Host.CheckValue(inputSchema, nameof(inputSchema));
 if (!inputSchema.TryFindColumn(_columnName, out SchemaShape.Column outCol) && outCol.ItemType != TextDataViewType.Instance)
 {
 throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", _columnName);
 }

 return inputSchema;
 }
 }

 // Internal only transformer used to facilitate the expansion of ngrams with pre-defined weights
 internal sealed class TextExpandingTransformer : RowToRowTransformerBase
 {
 internal const string Summary = "Expands text in the format of term:freq; to have the correct number of terms";
 internal const string UserName = "Text Expanding Transform";
 internal const string LoadName = "TextExpand";

 internal const string LoaderSignature = "TextExpandTransform";

 private readonly string _columnName;
 private readonly char _freqSeparator;
 private readonly char _termSeparator;

 public TextExpandingTransformer(IHostEnvironment env, string columnName, char freqSeparator, char termSeparator)
 : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(TextExpandingTransformer)))
 {
 _columnName = columnName;
 _freqSeparator = freqSeparator;
 _termSeparator = termSeparator;
 }

 private static VersionInfo GetVersionInfo()
 {
 return new VersionInfo(
 modelSignature: "TEXT EXP",
 verWrittenCur: 0x00010001, // Initial
 verReadableCur: 0x00010001,
 verWeCanReadBack: 0x00010001,
 loaderSignature: LoaderSignature,
 loaderAssemblyName: typeof(TextExpandingTransformer).Assembly.FullName);
 }

 /// <summary>
 /// Factory method for SignatureLoadModel.
 /// </summary>
 private TextExpandingTransformer(IHostEnvironment env, ModelLoadContext ctx) :
 base(Contracts.CheckRef(env, nameof(env)).Register(nameof(ColumnConcatenatingTransformer)))
 {
 Host.CheckValue(ctx, nameof(ctx));
 ctx.CheckAtModel(GetVersionInfo());
 // *** Binary format ***
 // string: column n ame
 // char: frequency separator
 // char: term separator

 _columnName = ctx.Reader.ReadString();
 _freqSeparator = ctx.Reader.ReadChar();
 _termSeparator = ctx.Reader.ReadChar();
 }

 /// <summary>
 /// Factory method for SignatureLoadRowMapper.
 /// </summary>
 private static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, DataViewSchema inputSchema)
 => new TextExpandingTransformer(env, ctx).MakeRowMapper(inputSchema);

 /// <summary>
 /// Factory method for SignatureLoadDataTransform.
 /// </summary>
 private static IDataTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
 => new TextExpandingTransformer(env, ctx).MakeDataTransform(input);

 private protected override IRowMapper MakeRowMapper(DataViewSchema schema)
 {
 return new Mapper(Host, schema, this);
 }

 private protected override void SaveModel(ModelSaveContext ctx)
 {
 Host.CheckValue(ctx, nameof(ctx));
 ctx.CheckAtModel();
 ctx.SetVersionInfo(GetVersionInfo());

 // *** Binary format ***
 // string: column n ame
 // char: frequency separator
 // char: term separator

 ctx.Writer.Write(_columnName);
 ctx.Writer.Write(_freqSeparator);
 ctx.Writer.Write(_termSeparator);
 }

 private sealed class Mapper : MapperBase
 {
 private readonly TextExpandingTransformer _parent;
 public Mapper(IHost host, DataViewSchema inputSchema, RowToRowTransformerBase parent)
 : base(host, inputSchema, parent)
 {
 _parent = (TextExpandingTransformer)parent;
 }

 protected override DataViewSchema.DetachedColumn[] GetOutputColumnsCore()
 {
 return new DataViewSchema.DetachedColumn[]
 {
 new DataViewSchema.DetachedColumn(_parent._columnName, TextDataViewType.Instance)
 };
 }

 protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func<int, bool> activeOutput, out Action disposer)
 {
 disposer = null;
 ValueGetter<ReadOnlyMemory<char>> srcGetter = input.GetGetter<ReadOnlyMemory<char>>(input.Schema.GetColumnOrNull(_parent._columnName).Value);
 ReadOnlyMemory<char> inputMem = default;
 var sb = new StringBuilder();

 ValueGetter<ReadOnlyMemory<char>> result = (ref ReadOnlyMemory<char> dst) =>
 {
 sb.Clear();
 srcGetter(ref inputMem);
 var inputText = inputMem.ToString();
 foreach (var termFreq in inputText.Split(_parent._termSeparator))
 {
 var tf = termFreq.Split(_parent._freqSeparator);
 if (tf.Length != 2)
 sb.Append(tf[0] + " ");
 else
 {
 for (int i = 0; i < int.Parse(tf[1]); i++)
 sb.Append(tf[0] + " ");
 }
 }

 dst = sb.ToString().AsMemory();
 };

 return result;
 }

 private protected override Func<int, bool> GetDependenciesCore(Func<int, bool> activeOutput)
 {
 var active = new bool[InputSchema.Count];
 if (activeOutput(0))
 {
 active[InputSchema.GetColumnOrNull(_parent._columnName).Value.Index] = true;
 }
 return col => active[col];
 }

 private protected override void SaveModel(ModelSaveContext ctx)
 {
 _parent.SaveModel(ctx);
 }
 }
 }

 #endregion TextExpander
 }

 /// <summary>
Expand DownExpand Up@@ -235,6 +424,13 @@ internal abstract class ArgumentsBase

 [Argument(ArgumentType.AtMostOnce, HelpText = "The weighting criteria")]
 public NgramExtractingEstimator.WeightingCriteria Weighting = NgramExtractingEstimator.Defaults.Weighting;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Separator used to separate terms/frequency pairs.")]
 public char TermSeparator = default;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Separator used to separate terms from their frequency.")]
 public char FreqSeparator = default;

 }

 [TlcModule.Component(Name = "NGram", FriendlyName = "NGram Extractor Transform", Alias = "NGramExtractorTransform,NGramExtractor",
Expand Down

30 changes: 24 additions & 6 deletions
src/Microsoft.ML.Transforms/Text/WrappedTextTransformers.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,6 +45,8 @@ public sealed class WordBagEstimator : IEstimator<ITransformer>
 private readonly bool _useAllLengths;
 private readonly int _maxNumTerms;
 private readonly NgramExtractingEstimator.WeightingCriteria _weighting;
 private readonly char _termSeparator;
 private readonly char _freqSeparator;

 /// <summary>
 /// Options for how the n-grams are extracted.
Expand DownExpand Up@@ -99,15 +101,19 @@ public Options()
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 string outputColumnName,
 string inputColumnName = null,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 : this(env, outputColumnName, new[] { inputColumnName ?? outputColumnName }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 : this(env, outputColumnName, new[] { inputColumnName ?? outputColumnName }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting, termSeparator, freqSeparator)
 {
 }

Expand All@@ -123,15 +129,19 @@ internal WordBagEstimator(IHostEnvironment env,
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 string outputColumnName,
 string[] inputColumnNames,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 : this(env, new[] { (outputColumnName, inputColumnNames) }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 : this(env, new[] { (outputColumnName, inputColumnNames) }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting, termSeparator, freqSeparator)
 {
 }

Expand All@@ -146,13 +156,17 @@ internal WordBagEstimator(IHostEnvironment env,
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 (string outputColumnName, string[] inputColumnNames)[] columns,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 {
 Contracts.CheckValue(env, nameof(env));
 _host = env.Register(nameof(WordBagEstimator));
Expand All@@ -169,6 +183,8 @@ internal WordBagEstimator(IHostEnvironment env,
 _useAllLengths = useAllLengths;
 _maxNumTerms = maximumNgramsCount;
 _weighting = weighting;
 _termSeparator = termSeparator;
 _freqSeparator = freqSeparator;
 }

 /// <summary> Trains and returns a <see cref="ITransformer"/>.</summary>
Expand All@@ -187,7 +203,9 @@ private WordBagBuildingTransformer.Options CreateOptions()
 SkipLength = _skipLength,
 UseAllLengths = _useAllLengths,
 MaxNumTerms = new[] { _maxNumTerms },
 Weighting = _weighting
 Weighting = _weighting,
 TermSeparator = _termSeparator,
 FreqSeparator = _freqSeparator,
 };
 }

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Add in ability to have pre-defined weights for ngrams.#6458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

michaelgsharp
merged 5 commits into

from

Nov 23, 2022
 Merged


Add in ability to have pre-defined weights for ngrams.
#6458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
b91fe6f

ngram pre defined weights


michaelgsharp Nov 17, 2022
0bed831

removed commented code


michaelgsharp Nov 17, 2022
a436e61

formatting fix


michaelgsharp Nov 17, 2022
88d7557

renegerate entry points


michaelgsharp Nov 17, 2022
61044f5

fixing API


michaelgsharp Nov 23, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


25 changes: 25 additions & 0 deletions
src/Microsoft.ML.Transforms/Text/TextCatalog.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -362,6 +362,31 @@ public static WordBagEstimator ProduceWordBags(this TransformsCatalog.TextTransf
 => new WordBagEstimator(Contracts.CheckRef(catalog, nameof(catalog)).GetEnvironment(),
 outputColumnName, inputColumnName, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting);

 /// <summary>
 /// Create a <see cref="WordBagEstimator"/>, which maps the column specified in <paramref name="inputColumnName"/>
 /// to a vector of n-gram counts in a new column named <paramref name="outputColumnName"/>.
 /// </summary>
 /// <remarks>
 /// <see cref="WordBagEstimator"/> is different from <see cref="NgramExtractingEstimator"/> in that the former
 /// tokenizes text internally and the latter takes tokenized text as input.
 /// </remarks>
 /// <param name="catalog">The transform's catalog.</param>
 /// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.
 /// This column's data type will be known-size vector of <see cref="System.Single"/>.</param>
 /// <param name="inputColumnName">Name of the column to take the data from.
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 /// This estimator operates over vector of text.</param>
 public static WordBagEstimator ProduceWordBags(this TransformsCatalog.TextTransforms catalog,
 string outputColumnName,
 char termSeparator,
 char freqSeparator,
 string inputColumnName = null,
 int maximumNgramsCount = NgramExtractingEstimator.Defaults.MaximumNgramsCount)
 => new WordBagEstimator(Contracts.CheckRef(catalog, nameof(catalog)).GetEnvironment(),
 outputColumnName, inputColumnName, 1, 0, true, maximumNgramsCount, NgramExtractingEstimator.WeightingCriteria.Tf, termSeparator: termSeparator, freqSeparator: freqSeparator);

 /// <summary>
 /// Create a <see cref="WordBagEstimator"/>, which maps the multiple columns specified in <paramref name="inputColumnNames"/>
 /// to a vector of n-gram counts in a new column named <paramref name="outputColumnName"/>.
Expand Down

200 changes: 198 additions & 2 deletions
src/Microsoft.ML.Transforms/Text/WordBagTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
Expand All@@ -12,6 +13,7 @@
using Microsoft.ML.Internal.Utilities;
using Microsoft.ML.Runtime;
using Microsoft.ML.Transforms.Text;
using static Microsoft.ML.Transforms.Text.WordBagBuildingTransformer;

[assembly: LoadableClass(WordBagBuildingTransformer.Summary, typeof(IDataTransform), typeof(WordBagBuildingTransformer), typeof(WordBagBuildingTransformer.Options), typeof(SignatureDataTransform),
 "Word Bag Transform", "WordBagTransform", "WordBag")]
Expand All@@ -21,6 +23,16 @@

[assembly: EntryPointModule(typeof(NgramExtractorTransform.NgramExtractorArguments))]

// These are for the internal only TextExpandingTransformer. Not exposed publically
[assembly: LoadableClass(TextExpandingTransformer.Summary, typeof(IDataTransform), typeof(TextExpandingTransformer), null, typeof(SignatureLoadDataTransform),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

[assembly: LoadableClass(typeof(TextExpandingTransformer), null, typeof(SignatureLoadModel),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

[assembly: LoadableClass(typeof(IRowMapper), typeof(TextExpandingTransformer), null, typeof(SignatureLoadRowMapper),
 TextExpandingTransformer.UserName, TextExpandingTransformer.LoaderSignature)]

namespace Microsoft.ML.Transforms.Text
{
 /// <summary>
Expand DownExpand Up@@ -144,18 +156,195 @@ internal static IEstimator<ITransformer> CreateEstimator(IHostEnvironment env, O
 NgramLength = column.NgramLength,
 SkipLength = column.SkipLength,
 Weighting = column.Weighting,
 UseAllLengths = column.UseAllLengths
 UseAllLengths = column.UseAllLengths,
 };
 }

 IEstimator<ITransformer> estimator = NgramExtractionUtils.GetConcatEstimator(h, options.Columns);
 estimator = estimator.Append(new WordTokenizingEstimator(env, tokenizeColumns));
 if (options.FreqSeparator != default)
 {
 estimator = estimator.Append(new TextExpandingEstimator(h, tokenizeColumns[0].InputColumnName, options.FreqSeparator, options.TermSeparator));
 }
 estimator = estimator.Append(new WordTokenizingEstimator(h, tokenizeColumns));
 estimator = estimator.Append(NgramExtractorTransform.CreateEstimator(h, extractorArgs, estimator.GetOutputSchema(inputSchema)));
 return estimator;
 }

 internal static IDataTransform Create(IHostEnvironment env, Options options, IDataView input) =>
 (IDataTransform)CreateEstimator(env, options, SchemaShape.Create(input.Schema)).Fit(input).Transform(input);

 #region TextExpander

 // Internal only estimator used to facilitate the expansion of ngrams with pre-defined weights
 internal sealed class TextExpandingEstimator : TrivialEstimator<TextExpandingTransformer>
 {
 private readonly string _columnName;
 public TextExpandingEstimator(IHostEnvironment env, string columnName, char freqSeparator, char termSeparator)
 : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(TextExpandingEstimator)), new TextExpandingTransformer(env, columnName, freqSeparator, termSeparator))
 {
 _columnName = columnName;
 }

 public override SchemaShape GetOutputSchema(SchemaShape inputSchema)
 {
 Host.CheckValue(inputSchema, nameof(inputSchema));
 if (!inputSchema.TryFindColumn(_columnName, out SchemaShape.Column outCol) && outCol.ItemType != TextDataViewType.Instance)
 {
 throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", _columnName);
 }

 return inputSchema;
 }
 }

 // Internal only transformer used to facilitate the expansion of ngrams with pre-defined weights
 internal sealed class TextExpandingTransformer : RowToRowTransformerBase
 {
 internal const string Summary = "Expands text in the format of term:freq; to have the correct number of terms";
 internal const string UserName = "Text Expanding Transform";
 internal const string LoadName = "TextExpand";

 internal const string LoaderSignature = "TextExpandTransform";

 private readonly string _columnName;
 private readonly char _freqSeparator;
 private readonly char _termSeparator;

 public TextExpandingTransformer(IHostEnvironment env, string columnName, char freqSeparator, char termSeparator)
 : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(TextExpandingTransformer)))
 {
 _columnName = columnName;
 _freqSeparator = freqSeparator;
 _termSeparator = termSeparator;
 }

 private static VersionInfo GetVersionInfo()
 {
 return new VersionInfo(
 modelSignature: "TEXT EXP",
 verWrittenCur: 0x00010001, // Initial
 verReadableCur: 0x00010001,
 verWeCanReadBack: 0x00010001,
 loaderSignature: LoaderSignature,
 loaderAssemblyName: typeof(TextExpandingTransformer).Assembly.FullName);
 }

 /// <summary>
 /// Factory method for SignatureLoadModel.
 /// </summary>
 private TextExpandingTransformer(IHostEnvironment env, ModelLoadContext ctx) :
 base(Contracts.CheckRef(env, nameof(env)).Register(nameof(ColumnConcatenatingTransformer)))
 {
 Host.CheckValue(ctx, nameof(ctx));
 ctx.CheckAtModel(GetVersionInfo());
 // *** Binary format ***
 // string: column n ame
 // char: frequency separator
 // char: term separator

 _columnName = ctx.Reader.ReadString();
 _freqSeparator = ctx.Reader.ReadChar();
 _termSeparator = ctx.Reader.ReadChar();
 }

 /// <summary>
 /// Factory method for SignatureLoadRowMapper.
 /// </summary>
 private static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, DataViewSchema inputSchema)
 => new TextExpandingTransformer(env, ctx).MakeRowMapper(inputSchema);

 /// <summary>
 /// Factory method for SignatureLoadDataTransform.
 /// </summary>
 private static IDataTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
 => new TextExpandingTransformer(env, ctx).MakeDataTransform(input);

 private protected override IRowMapper MakeRowMapper(DataViewSchema schema)
 {
 return new Mapper(Host, schema, this);
 }

 private protected override void SaveModel(ModelSaveContext ctx)
 {
 Host.CheckValue(ctx, nameof(ctx));
 ctx.CheckAtModel();
 ctx.SetVersionInfo(GetVersionInfo());

 // *** Binary format ***
 // string: column n ame
 // char: frequency separator
 // char: term separator

 ctx.Writer.Write(_columnName);
 ctx.Writer.Write(_freqSeparator);
 ctx.Writer.Write(_termSeparator);
 }

 private sealed class Mapper : MapperBase
 {
 private readonly TextExpandingTransformer _parent;
 public Mapper(IHost host, DataViewSchema inputSchema, RowToRowTransformerBase parent)
 : base(host, inputSchema, parent)
 {
 _parent = (TextExpandingTransformer)parent;
 }

 protected override DataViewSchema.DetachedColumn[] GetOutputColumnsCore()
 {
 return new DataViewSchema.DetachedColumn[]
 {
 new DataViewSchema.DetachedColumn(_parent._columnName, TextDataViewType.Instance)
 };
 }

 protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func<int, bool> activeOutput, out Action disposer)
 {
 disposer = null;
 ValueGetter<ReadOnlyMemory<char>> srcGetter = input.GetGetter<ReadOnlyMemory<char>>(input.Schema.GetColumnOrNull(_parent._columnName).Value);
 ReadOnlyMemory<char> inputMem = default;
 var sb = new StringBuilder();

 ValueGetter<ReadOnlyMemory<char>> result = (ref ReadOnlyMemory<char> dst) =>
 {
 sb.Clear();
 srcGetter(ref inputMem);
 var inputText = inputMem.ToString();
 foreach (var termFreq in inputText.Split(_parent._termSeparator))
 {
 var tf = termFreq.Split(_parent._freqSeparator);
 if (tf.Length != 2)
 sb.Append(tf[0] + " ");
 else
 {
 for (int i = 0; i < int.Parse(tf[1]); i++)
 sb.Append(tf[0] + " ");
 }
 }

 dst = sb.ToString().AsMemory();
 };

 return result;
 }

 private protected override Func<int, bool> GetDependenciesCore(Func<int, bool> activeOutput)
 {
 var active = new bool[InputSchema.Count];
 if (activeOutput(0))
 {
 active[InputSchema.GetColumnOrNull(_parent._columnName).Value.Index] = true;
 }
 return col => active[col];
 }

 private protected override void SaveModel(ModelSaveContext ctx)
 {
 _parent.SaveModel(ctx);
 }
 }
 }

 #endregion TextExpander
 }

 /// <summary>
Expand DownExpand Up@@ -235,6 +424,13 @@ internal abstract class ArgumentsBase

 [Argument(ArgumentType.AtMostOnce, HelpText = "The weighting criteria")]
 public NgramExtractingEstimator.WeightingCriteria Weighting = NgramExtractingEstimator.Defaults.Weighting;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Separator used to separate terms/frequency pairs.")]
 public char TermSeparator = default;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Separator used to separate terms from their frequency.")]
 public char FreqSeparator = default;

 }

 [TlcModule.Component(Name = "NGram", FriendlyName = "NGram Extractor Transform", Alias = "NGramExtractorTransform,NGramExtractor",
Expand Down

30 changes: 24 additions & 6 deletions
src/Microsoft.ML.Transforms/Text/WrappedTextTransformers.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,6 +45,8 @@ public sealed class WordBagEstimator : IEstimator<ITransformer>
 private readonly bool _useAllLengths;
 private readonly int _maxNumTerms;
 private readonly NgramExtractingEstimator.WeightingCriteria _weighting;
 private readonly char _termSeparator;
 private readonly char _freqSeparator;

 /// <summary>
 /// Options for how the n-grams are extracted.
Expand DownExpand Up@@ -99,15 +101,19 @@ public Options()
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 string outputColumnName,
 string inputColumnName = null,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 : this(env, outputColumnName, new[] { inputColumnName ?? outputColumnName }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 : this(env, outputColumnName, new[] { inputColumnName ?? outputColumnName }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting, termSeparator, freqSeparator)
 {
 }

Expand All@@ -123,15 +129,19 @@ internal WordBagEstimator(IHostEnvironment env,
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 string outputColumnName,
 string[] inputColumnNames,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 : this(env, new[] { (outputColumnName, inputColumnNames) }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 : this(env, new[] { (outputColumnName, inputColumnNames) }, ngramLength, skipLength, useAllLengths, maximumNgramsCount, weighting, termSeparator, freqSeparator)
 {
 }

Expand All@@ -146,13 +156,17 @@ internal WordBagEstimator(IHostEnvironment env,
 /// <param name="useAllLengths">Whether to include all n-gram lengths up to <paramref name="ngramLength"/> or only <paramref name="ngramLength"/>.</param>
 /// <param name="maximumNgramsCount">Maximum number of n-grams to store in the dictionary.</param>
 /// <param name="weighting">Statistical measure used to evaluate how important a word is to a document in a corpus.</param>
 /// <param name="termSeparator">Separator used to separate terms/frequency pairs.</param>
 /// <param name="freqSeparator">Separator used to separate terms from their frequency.</param>
 internal WordBagEstimator(IHostEnvironment env,
 (string outputColumnName, string[] inputColumnNames)[] columns,
 int ngramLength = 1,
 int skipLength = 0,
 bool useAllLengths = true,
 int maximumNgramsCount = 10000000,
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf)
 NgramExtractingEstimator.WeightingCriteria weighting = NgramExtractingEstimator.WeightingCriteria.Tf,
 char termSeparator = default,
 char freqSeparator = default)
 {
 Contracts.CheckValue(env, nameof(env));
 _host = env.Register(nameof(WordBagEstimator));
Expand All@@ -169,6 +183,8 @@ internal WordBagEstimator(IHostEnvironment env,
 _useAllLengths = useAllLengths;
 _maxNumTerms = maximumNgramsCount;
 _weighting = weighting;
 _termSeparator = termSeparator;
 _freqSeparator = freqSeparator;
 }

 /// <summary> Trains and returns a <see cref="ITransformer"/>.</summary>
Expand All@@ -187,7 +203,9 @@ private WordBagBuildingTransformer.Options CreateOptions()
 SkipLength = _skipLength,
 UseAllLengths = _useAllLengths,
 MaxNumTerms = new[] { _maxNumTerms },
 Weighting = _weighting
 Weighting = _weighting,
 TermSeparator = _termSeparator,
 FreqSeparator = _freqSeparator,
 };
 }

Expand Down
Loading