Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions src/Microsoft.ML.Data/Transforms/ConcatTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -90,6 +90,19 @@ public bool TryUnparse(StringBuilder sb)

public sealed class Arguments : TransformInputBase
{
public Arguments()
{
}

public Arguments(string name, params string[] source)
{
Column = new[] { new Column()
{
Name = name,
Source = source
}};
}

[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:srcs)", ShortName = "col", SortOrder = 1)]
public Column[] Column;
}
Expand DownExpand Up@@ -527,6 +540,18 @@ private static VersionInfo GetVersionInfo()

public override ISchema Schema => _bindings;

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Input columns to concatenate.</param>
public ConcatTransform(IHostEnvironment env, IDataView input, string name, params string[] source)
: this(env, new Arguments(name, source), input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand Down
12 changes: 12 additions & 0 deletions src/Microsoft.ML.Data/Transforms/CopyColumnsTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,6 +64,18 @@ private static VersionInfo GetVersionInfo()

private const string RegistrationName = "CopyColumns";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be copied.</param>
public CopyColumnsTransform(IHostEnvironment env, IDataView input, string name, string source)
: this(env, new Arguments(){ Column = new[] { new Column() { Source = source, Name = name }}}, input)
{
}

public CopyColumnsTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, null)
{
Expand Down
24 changes: 24 additions & 0 deletions src/Microsoft.ML.Data/Transforms/DropColumnsTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -237,6 +237,17 @@ private static VersionInfo GetVersionInfo()
private const string DropRegistrationName = "DropColumns";
private const string KeepRegistrationName = "KeepColumns";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="columnsToDrop">Name of the columns to be dropped.</param>
public DropColumnsTransform(IHostEnvironment env, IDataView input, params string[] columnsToDrop)
:this(env, new Arguments() { Column = columnsToDrop }, input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand DownExpand Up@@ -383,4 +394,17 @@ public ValueGetter<TValue> GetGetter<TValue>(int col)
}
}
}

public class KeepColumnsTransform
{
/// <summary>
/// A helper method to create <see cref="KeepColumnsTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="columnsToKeep">Name of the columns to be kept. All other columns will be removed.</param>
/// <returns></returns>
public static IDataTransform Create(IHostEnvironment env, IDataView input, params string[] columnsToKeep)
=> new DropColumnsTransform(env, new DropColumnsTransform.KeepArguments() { Column = columnsToKeep }, input);
}
}
19 changes: 18 additions & 1 deletion src/Microsoft.ML.Data/Transforms/NAFilter.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,13 +28,18 @@ namespace Microsoft.ML.Runtime.Data
{
public sealed class NAFilter : FilterBase
{
private static class Defaults
{
public const bool Complement = false;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Column", ShortName = "col", SortOrder = 1)]
public string[] Column;

[Argument(ArgumentType.Multiple, HelpText = "If true, keep only rows that contain NA values, and filter the rest.")]
public bool Complement;
public bool Complement = Defaults.Complement;
}

private sealed class ColInfo
Expand DownExpand Up@@ -72,6 +77,18 @@ private static VersionInfo GetVersionInfo()
private readonly bool _complement;
private const string RegistrationName = "MissingValueFilter";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="complement">If true, keep only rows that contain NA values, and filter the rest.</param>
/// <param name="columns">Name of the columns. Only these columns will be used to filter rows having 'NA' values.</param>
public NAFilter(IHostEnvironment env, IDataView input, bool complement = Defaults.Complement, params string[] columns)
: this(env, new Arguments() { Column = columns, Complement = complement }, input)
{
}

public NAFilter(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, input)
{
Expand Down
32 changes: 29 additions & 3 deletions src/Microsoft.ML.Transforms/BootstrapSampleTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,20 +25,27 @@ namespace Microsoft.ML.Runtime.Data
/// </summary>
public sealed class BootstrapSampleTransform : FilterBase
{
private static class Defaults
{
public const bool Complement = false;
public const bool ShuffleInput = true;
public const int PoolSize = 1000;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.AtMostOnce, HelpText = "Whether this is the out-of-bag sample, that is, all those rows that are not selected by the transform.",
ShortName = "comp")]
public bool Complement;
public bool Complement = Defaults.Complement;

[Argument(ArgumentType.AtMostOnce, HelpText = "The random seed. If unspecified random state will be instead derived from the environment.")]
public uint? Seed;

[Argument(ArgumentType.AtMostOnce, HelpText = "Whether we should attempt to shuffle the source data. By default on, but can be turned off for efficiency.", ShortName = "si")]
public bool ShuffleInput = true;
public bool ShuffleInput = Defaults.ShuffleInput;

[Argument(ArgumentType.LastOccurenceWins, HelpText = "When shuffling the output, the number of output rows to keep in that pool. Note that shuffling of output is completely distinct from shuffling of input.", ShortName = "pool")]
public int PoolSize = 1000;
public int PoolSize = Defaults.PoolSize;
}

internal const string Summary = "Approximate bootstrap sampling.";
Expand DownExpand Up@@ -76,6 +83,25 @@ public BootstrapSampleTransform(IHostEnvironment env, Arguments args, IDataView
_poolSize = args.PoolSize;
}

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="complement">Whether this is the out-of-bag sample, that is, all those rows that are not selected by the transform.</param>
/// <param name="seed">The random seed. If unspecified random state will be instead derived from the environment.</param>
/// <param name="shuffleInput">Whether we should attempt to shuffle the source data. By default on, but can be turned off for efficiency.</param>
/// <param name="poolSize">When shuffling the output, the number of output rows to keep in that pool. Note that shuffling of output is completely distinct from shuffling of input.</param>
public BootstrapSampleTransform(IHostEnvironment env,
IDataView input,
bool complement = Defaults.Complement,
uint? seed = null,
bool shuffleInput = Defaults.ShuffleInput,
int poolSize = Defaults.PoolSize)
: this(env, new Arguments() { Complement = complement, Seed = seed, ShuffleInput = shuffleInput, PoolSize = poolSize }, input)
{
}

private BootstrapSampleTransform(IHost host, ModelLoadContext ctx, IDataView input)
: base(host, input)
{
Expand Down
51 changes: 46 additions & 5 deletions src/Microsoft.ML.Transforms/CategoricalHashTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -86,6 +86,15 @@ public bool TryUnparse(StringBuilder sb)
}
}

private static class Defaults
{
public const int HashBits = 16;
public const uint Seed = 314489979;
public const bool Ordered = true;
public const int InvertHash = 0;
public const CategoricalTransform.OutputKind OutputKind = CategoricalTransform.OutputKind.Bag;
}

/// <summary>
/// This class is a merger of <see cref="HashTransform.Arguments"/> and <see cref="KeyToVectorTransform.Arguments"/>
/// with join option removed
Expand All@@ -97,29 +106,61 @@ public sealed class Arguments : TransformInputBase

[Argument(ArgumentType.AtMostOnce, HelpText = "Number of bits to hash into. Must be between 1 and 30, inclusive.",
ShortName = "bits", SortOrder = 2)]
public int HashBits = 16;
public int HashBits = Defaults.HashBits;

[Argument(ArgumentType.AtMostOnce, HelpText = "Hashing seed")]
public uint Seed = 314489979;
public uint Seed = Defaults.Seed;

[Argument(ArgumentType.AtMostOnce, HelpText = "Whether the position of each term should be included in the hash", ShortName = "ord")]
public bool Ordered = true;
public bool Ordered = Defaults.Ordered;

[Argument(ArgumentType.AtMostOnce,
HelpText = "Limit the number of keys used to generate the slot name to this many. 0 means no invert hashing, -1 means no limit.",
ShortName = "ih")]
public int InvertHash;
public int InvertHash = Defaults.InvertHash;

[Argument(ArgumentType.AtMostOnce, HelpText = "Output kind: Bag (multi-set vector), Ind (indicator vector), or Key (index)",
ShortName = "kind", SortOrder = 102)]
public CategoricalTransform.OutputKind OutputKind = CategoricalTransform.OutputKind.Bag;
public CategoricalTransform.OutputKind OutputKind = Defaults.OutputKind;
}

internal const string Summary = "Converts the categorical value into an indicator array by hashing the value and using the hash as an index in the "
+ "bag. If the input column is a vector, a single indicator bag is returned for it.";

public const string UserName = "Categorical Hash Transform";

/// <summary>
/// A helper method to create <see cref="CategoricalHashTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
/// <param name="hashBits">Number of bits to hash into. Must be between 1 and 30, inclusive.</param>
/// <param name="invertHash">Limit the number of keys used to generate the slot name to this many. 0 means no invert hashing, -1 means no limit.</param>
/// <param name="outputKind">The type of output expected.</param>
public static IDataTransform Create(IHostEnvironment env,
IDataView input,
string name,
string source =null,
int hashBits = Defaults.HashBits,
int invertHash = Defaults.InvertHash,
CategoricalTransform.OutputKind outputKind = Defaults.OutputKind)
{
var args = new Arguments()
{
Column = new[] { new Column(){
Source = source ?? name,
Name = name
}
},
HashBits = hashBits,
InvertHash = invertHash,
OutputKind = outputKind
};
return Create(env, args, input);
}

public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
Expand Down
41 changes: 40 additions & 1 deletion src/Microsoft.ML.Transforms/CategoricalTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,15 +38,27 @@ public static class CategoricalTransform
{
public enum OutputKind : byte
{
/// <summary>
/// Output is a bag (multi-set) vector
/// </summary>
[TGUI(Label = "Output is a bag (multi-set) vector")]
Bag = 1,

/// <summary>
/// Output is an indicator vector
/// </summary>
[TGUI(Label = "Output is an indicator vector")]
Ind = 2,

/// <summary>
/// Output is a key value
/// </summary>
[TGUI(Label = "Output is a key value")]
Key = 3,

/// <summary>
/// Output is binary encoded
/// </summary>
[TGUI(Label = "Output is binary encoded")]
Bin = 4,
}
Expand DownExpand Up@@ -96,14 +108,19 @@ public bool TryUnparse(StringBuilder sb)
}
}

private static class Defaults
{
public const OutputKind OutKind = OutputKind.Ind;
}

public sealed class Arguments : TermTransform.ArgumentsBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)]
public Column[] Column;

[Argument(ArgumentType.AtMostOnce, HelpText = "Output kind: Bag (multi-set vector), Ind (indicator vector), or Key (index)",
ShortName = "kind", SortOrder = 102)]
public OutputKind OutputKind = OutputKind.Ind;
public OutputKind OutputKind = Defaults.OutKind;

public Arguments()
{
Expand All@@ -118,6 +135,28 @@ public Arguments()

public const string UserName = "Categorical Transform";

/// <summary>
/// A helper method to create <see cref="CategoricalTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
/// <param name="outputKind">The type of output expected.</param>
public static IDataTransform Create(IHostEnvironment env, IDataView input, string name, string source = null, OutputKind outputKind = Defaults.OutKind)
{
var args = new Arguments()
{
Column = new[] { new Column(){
Source = source ?? name,
Name = name
}
},
OutputKind = outputKind
};
return Create(env, args, input);
}

public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
Expand Down
25 changes: 24 additions & 1 deletion src/Microsoft.ML.Transforms/CountFeatureSelection.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,17 +28,40 @@ public static class CountFeatureSelectionTransform
public const string Summary = "Selects the slots for which the count of non-default values is greater than or equal to a threshold.";
public const string UserName = "Count Feature Selection Transform";

private static class Defaults
{
public const long Count = 1;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Columns to use for feature selection", ShortName = "col", SortOrder = 1)]
public string[] Column;

[Argument(ArgumentType.Required, HelpText = "If the count of non-default values for a slot is greater than or equal to this threshold, the slot is preserved", ShortName = "c", SortOrder = 1)]
public long Count = 1;
public long Count = Defaults.Count;
}

internal static string RegistrationName = "CountFeatureSelectionTransform";

/// <summary>
/// A helper method to create CountFeatureSelection transform for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="count">If the count of non-default values for a slot is greater than or equal to this threshold, the slot is preserved.</param>
/// <param name="columns">Columns to use for feature selection.</param>
/// <returns></returns>
public static IDataTransform Create(IHostEnvironment env, IDataView input, long count = Defaults.Count, params string[] columns)
{
var args = new Arguments()
{
Column = columns,
Count = count
};
return Create(env, args, input);
}

/// <summary>
/// Create method corresponding to SignatureDataTransform.
/// </summary>
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions src/Microsoft.ML.Data/Transforms/ConcatTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -90,6 +90,19 @@ public bool TryUnparse(StringBuilder sb)

public sealed class Arguments : TransformInputBase
{
public Arguments()
{
}

public Arguments(string name, params string[] source)
{
Column = new[] { new Column()
{
Name = name,
Source = source
}};
}

[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:srcs)", ShortName = "col", SortOrder = 1)]
public Column[] Column;
}
Expand DownExpand Up@@ -527,6 +540,18 @@ private static VersionInfo GetVersionInfo()

public override ISchema Schema => _bindings;

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Input columns to concatenate.</param>
public ConcatTransform(IHostEnvironment env, IDataView input, string name, params string[] source)
: this(env, new Arguments(name, source), input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand Down
12 changes: 12 additions & 0 deletions src/Microsoft.ML.Data/Transforms/CopyColumnsTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,6 +64,18 @@ private static VersionInfo GetVersionInfo()

private const string RegistrationName = "CopyColumns";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be copied.</param>
public CopyColumnsTransform(IHostEnvironment env, IDataView input, string name, string source)
: this(env, new Arguments(){ Column = new[] { new Column() { Source = source, Name = name }}}, input)
{
}

public CopyColumnsTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, null)
{
Expand Down
24 changes: 24 additions & 0 deletions src/Microsoft.ML.Data/Transforms/DropColumnsTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -237,6 +237,17 @@ private static VersionInfo GetVersionInfo()
private const string DropRegistrationName = "DropColumns";
private const string KeepRegistrationName = "KeepColumns";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="columnsToDrop">Name of the columns to be dropped.</param>
public DropColumnsTransform(IHostEnvironment env, IDataView input, params string[] columnsToDrop)
:this(env, new Arguments() { Column = columnsToDrop }, input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand DownExpand Up@@ -383,4 +394,17 @@ public ValueGetter<TValue> GetGetter<TValue>(int col)
}
}
}

public class KeepColumnsTransform
{
/// <summary>
/// A helper method to create <see cref="KeepColumnsTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="columnsToKeep">Name of the columns to be kept. All other columns will be removed.</param>
/// <returns></returns>
public static IDataTransform Create(IHostEnvironment env, IDataView input, params string[] columnsToKeep)
=> new DropColumnsTransform(env, new DropColumnsTransform.KeepArguments() { Column = columnsToKeep }, input);
}
}
19 changes: 18 additions & 1 deletion src/Microsoft.ML.Data/Transforms/NAFilter.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,13 +28,18 @@ namespace Microsoft.ML.Runtime.Data
{
public sealed class NAFilter : FilterBase
{
private static class Defaults
{
public const bool Complement = false;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Column", ShortName = "col", SortOrder = 1)]
public string[] Column;

[Argument(ArgumentType.Multiple, HelpText = "If true, keep only rows that contain NA values, and filter the rest.")]
public bool Complement;
public bool Complement = Defaults.Complement;
}

private sealed class ColInfo
Expand DownExpand Up@@ -72,6 +77,18 @@ private static VersionInfo GetVersionInfo()
private readonly bool _complement;
private const string RegistrationName = "MissingValueFilter";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="complement">If true, keep only rows that contain NA values, and filter the rest.</param>
/// <param name="columns">Name of the columns. Only these columns will be used to filter rows having 'NA' values.</param>
public NAFilter(IHostEnvironment env, IDataView input, bool complement = Defaults.Complement, params string[] columns)
: this(env, new Arguments() { Column = columns, Complement = complement }, input)
{
}

public NAFilter(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, input)
{
Expand Down
32 changes: 29 additions & 3 deletions src/Microsoft.ML.Transforms/BootstrapSampleTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,20 +25,27 @@ namespace Microsoft.ML.Runtime.Data
/// </summary>
public sealed class BootstrapSampleTransform : FilterBase
{
private static class Defaults
{
public const bool Complement = false;
public const bool ShuffleInput = true;
public const int PoolSize = 1000;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.AtMostOnce, HelpText = "Whether this is the out-of-bag sample, that is, all those rows that are not selected by the transform.",
ShortName = "comp")]
public bool Complement;
public bool Complement = Defaults.Complement;

[Argument(ArgumentType.AtMostOnce, HelpText = "The random seed. If unspecified random state will be instead derived from the environment.")]
public uint? Seed;

[Argument(ArgumentType.AtMostOnce, HelpText = "Whether we should attempt to shuffle the source data. By default on, but can be turned off for efficiency.", ShortName = "si")]
public bool ShuffleInput = true;
public bool ShuffleInput = Defaults.ShuffleInput;

[Argument(ArgumentType.LastOccurenceWins, HelpText = "When shuffling the output, the number of output rows to keep in that pool. Note that shuffling of output is completely distinct from shuffling of input.", ShortName = "pool")]
public int PoolSize = 1000;
public int PoolSize = Defaults.PoolSize;
}

internal const string Summary = "Approximate bootstrap sampling.";
Expand DownExpand Up@@ -76,6 +83,25 @@ public BootstrapSampleTransform(IHostEnvironment env, Arguments args, IDataView
_poolSize = args.PoolSize;
}

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="complement">Whether this is the out-of-bag sample, that is, all those rows that are not selected by the transform.</param>
/// <param name="seed">The random seed. If unspecified random state will be instead derived from the environment.</param>
/// <param name="shuffleInput">Whether we should attempt to shuffle the source data. By default on, but can be turned off for efficiency.</param>
/// <param name="poolSize">When shuffling the output, the number of output rows to keep in that pool. Note that shuffling of output is completely distinct from shuffling of input.</param>
public BootstrapSampleTransform(IHostEnvironment env,
IDataView input,
bool complement = Defaults.Complement,
uint? seed = null,
bool shuffleInput = Defaults.ShuffleInput,
int poolSize = Defaults.PoolSize)
: this(env, new Arguments() { Complement = complement, Seed = seed, ShuffleInput = shuffleInput, PoolSize = poolSize }, input)
{
}

private BootstrapSampleTransform(IHost host, ModelLoadContext ctx, IDataView input)
: base(host, input)
{
Expand Down
51 changes: 46 additions & 5 deletions src/Microsoft.ML.Transforms/CategoricalHashTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -86,6 +86,15 @@ public bool TryUnparse(StringBuilder sb)
}
}

private static class Defaults
{
public const int HashBits = 16;
public const uint Seed = 314489979;
public const bool Ordered = true;
public const int InvertHash = 0;
public const CategoricalTransform.OutputKind OutputKind = CategoricalTransform.OutputKind.Bag;
}

/// <summary>
/// This class is a merger of <see cref="HashTransform.Arguments"/> and <see cref="KeyToVectorTransform.Arguments"/>
/// with join option removed
Expand All@@ -97,29 +106,61 @@ public sealed class Arguments : TransformInputBase

[Argument(ArgumentType.AtMostOnce, HelpText = "Number of bits to hash into. Must be between 1 and 30, inclusive.",
ShortName = "bits", SortOrder = 2)]
public int HashBits = 16;
public int HashBits = Defaults.HashBits;

[Argument(ArgumentType.AtMostOnce, HelpText = "Hashing seed")]
public uint Seed = 314489979;
public uint Seed = Defaults.Seed;

[Argument(ArgumentType.AtMostOnce, HelpText = "Whether the position of each term should be included in the hash", ShortName = "ord")]
public bool Ordered = true;
public bool Ordered = Defaults.Ordered;

[Argument(ArgumentType.AtMostOnce,
HelpText = "Limit the number of keys used to generate the slot name to this many. 0 means no invert hashing, -1 means no limit.",
ShortName = "ih")]
public int InvertHash;
public int InvertHash = Defaults.InvertHash;

[Argument(ArgumentType.AtMostOnce, HelpText = "Output kind: Bag (multi-set vector), Ind (indicator vector), or Key (index)",
ShortName = "kind", SortOrder = 102)]
public CategoricalTransform.OutputKind OutputKind = CategoricalTransform.OutputKind.Bag;
public CategoricalTransform.OutputKind OutputKind = Defaults.OutputKind;
}

internal const string Summary = "Converts the categorical value into an indicator array by hashing the value and using the hash as an index in the "
+ "bag. If the input column is a vector, a single indicator bag is returned for it.";

public const string UserName = "Categorical Hash Transform";

/// <summary>
/// A helper method to create <see cref="CategoricalHashTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
/// <param name="hashBits">Number of bits to hash into. Must be between 1 and 30, inclusive.</param>
/// <param name="invertHash">Limit the number of keys used to generate the slot name to this many. 0 means no invert hashing, -1 means no limit.</param>
/// <param name="outputKind">The type of output expected.</param>
public static IDataTransform Create(IHostEnvironment env,
IDataView input,
string name,
string source =null,
int hashBits = Defaults.HashBits,
int invertHash = Defaults.InvertHash,
CategoricalTransform.OutputKind outputKind = Defaults.OutputKind)
{
var args = new Arguments()
{
Column = new[] { new Column(){
Source = source ?? name,
Name = name
}
},
HashBits = hashBits,
InvertHash = invertHash,
OutputKind = outputKind
};
return Create(env, args, input);
}

public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
Expand Down
41 changes: 40 additions & 1 deletion src/Microsoft.ML.Transforms/CategoricalTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,15 +38,27 @@ public static class CategoricalTransform
{
public enum OutputKind : byte
{
/// <summary>
/// Output is a bag (multi-set) vector
/// </summary>
[TGUI(Label = "Output is a bag (multi-set) vector")]
Bag = 1,

/// <summary>
/// Output is an indicator vector
/// </summary>
[TGUI(Label = "Output is an indicator vector")]
Ind = 2,

/// <summary>
/// Output is a key value
/// </summary>
[TGUI(Label = "Output is a key value")]
Key = 3,

/// <summary>
/// Output is binary encoded
/// </summary>
[TGUI(Label = "Output is binary encoded")]
Bin = 4,
}
Expand DownExpand Up@@ -96,14 +108,19 @@ public bool TryUnparse(StringBuilder sb)
}
}

private static class Defaults
{
public const OutputKind OutKind = OutputKind.Ind;
}

public sealed class Arguments : TermTransform.ArgumentsBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)]
public Column[] Column;

[Argument(ArgumentType.AtMostOnce, HelpText = "Output kind: Bag (multi-set vector), Ind (indicator vector), or Key (index)",
ShortName = "kind", SortOrder = 102)]
public OutputKind OutputKind = OutputKind.Ind;
public OutputKind OutputKind = Defaults.OutKind;

public Arguments()
{
Expand All@@ -118,6 +135,28 @@ public Arguments()

public const string UserName = "Categorical Transform";

/// <summary>
/// A helper method to create <see cref="CategoricalTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
/// <param name="outputKind">The type of output expected.</param>
public static IDataTransform Create(IHostEnvironment env, IDataView input, string name, string source = null, OutputKind outputKind = Defaults.OutKind)
{
var args = new Arguments()
{
Column = new[] { new Column(){
Source = source ?? name,
Name = name
}
},
OutputKind = outputKind
};
return Create(env, args, input);
}

public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
Expand Down
25 changes: 24 additions & 1 deletion src/Microsoft.ML.Transforms/CountFeatureSelection.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,17 +28,40 @@ public static class CountFeatureSelectionTransform
public const string Summary = "Selects the slots for which the count of non-default values is greater than or equal to a threshold.";
public const string UserName = "Count Feature Selection Transform";

private static class Defaults
{
public const long Count = 1;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Columns to use for feature selection", ShortName = "col", SortOrder = 1)]
public string[] Column;

[Argument(ArgumentType.Required, HelpText = "If the count of non-default values for a slot is greater than or equal to this threshold, the slot is preserved", ShortName = "c", SortOrder = 1)]
public long Count = 1;
public long Count = Defaults.Count;
}

internal static string RegistrationName = "CountFeatureSelectionTransform";

/// <summary>
/// A helper method to create CountFeatureSelection transform for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="count">If the count of non-default values for a slot is greater than or equal to this threshold, the slot is preserved.</param>
/// <param name="columns">Columns to use for feature selection.</param>
/// <returns></returns>
public static IDataTransform Create(IHostEnvironment env, IDataView input, long count = Defaults.Count, params string[] columns)
{
var args = new Arguments()
{
Column = columns,
Count = count
};
return Create(env, args, input);
}

/// <summary>
/// Create method corresponding to SignatureDataTransform.
/// </summary>
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix"); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions src/Microsoft.ML.Data/Transforms/ConcatTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -90,6 +90,19 @@ public bool TryUnparse(StringBuilder sb)

public sealed class Arguments : TransformInputBase
{
public Arguments()
{
}

public Arguments(string name, params string[] source)
{
Column = new[] { new Column()
{
Name = name,
Source = source
}};
}

[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:srcs)", ShortName = "col", SortOrder = 1)]
public Column[] Column;
}
Expand DownExpand Up@@ -527,6 +540,18 @@ private static VersionInfo GetVersionInfo()

public override ISchema Schema => _bindings;

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Input columns to concatenate.</param>
public ConcatTransform(IHostEnvironment env, IDataView input, string name, params string[] source)
: this(env, new Arguments(name, source), input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand Down
12 changes: 12 additions & 0 deletions src/Microsoft.ML.Data/Transforms/CopyColumnsTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,6 +64,18 @@ private static VersionInfo GetVersionInfo()

private const string RegistrationName = "CopyColumns";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be copied.</param>
public CopyColumnsTransform(IHostEnvironment env, IDataView input, string name, string source)
: this(env, new Arguments(){ Column = new[] { new Column() { Source = source, Name = name }}}, input)
{
}

public CopyColumnsTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, null)
{
Expand Down
24 changes: 24 additions & 0 deletions src/Microsoft.ML.Data/Transforms/DropColumnsTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -237,6 +237,17 @@ private static VersionInfo GetVersionInfo()
private const string DropRegistrationName = "DropColumns";
private const string KeepRegistrationName = "KeepColumns";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="columnsToDrop">Name of the columns to be dropped.</param>
public DropColumnsTransform(IHostEnvironment env, IDataView input, params string[] columnsToDrop)
:this(env, new Arguments() { Column = columnsToDrop }, input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand DownExpand Up@@ -383,4 +394,17 @@ public ValueGetter<TValue> GetGetter<TValue>(int col)
}
}
}

public class KeepColumnsTransform
{
/// <summary>
/// A helper method to create <see cref="KeepColumnsTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="columnsToKeep">Name of the columns to be kept. All other columns will be removed.</param>
/// <returns></returns>
public static IDataTransform Create(IHostEnvironment env, IDataView input, params string[] columnsToKeep)
=> new DropColumnsTransform(env, new DropColumnsTransform.KeepArguments() { Column = columnsToKeep }, input);
}
}
19 changes: 18 additions & 1 deletion src/Microsoft.ML.Data/Transforms/NAFilter.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,13 +28,18 @@ namespace Microsoft.ML.Runtime.Data
{
public sealed class NAFilter : FilterBase
{
private static class Defaults
{
public const bool Complement = false;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Column", ShortName = "col", SortOrder = 1)]
public string[] Column;

[Argument(ArgumentType.Multiple, HelpText = "If true, keep only rows that contain NA values, and filter the rest.")]
public bool Complement;
public bool Complement = Defaults.Complement;
}

private sealed class ColInfo
Expand DownExpand Up@@ -72,6 +77,18 @@ private static VersionInfo GetVersionInfo()
private readonly bool _complement;
private const string RegistrationName = "MissingValueFilter";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="complement">If true, keep only rows that contain NA values, and filter the rest.</param>
/// <param name="columns">Name of the columns. Only these columns will be used to filter rows having 'NA' values.</param>
public NAFilter(IHostEnvironment env, IDataView input, bool complement = Defaults.Complement, params string[] columns)
: this(env, new Arguments() { Column = columns, Complement = complement }, input)
{
}

public NAFilter(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, input)
{
Expand Down
32 changes: 29 additions & 3 deletions src/Microsoft.ML.Transforms/BootstrapSampleTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,20 +25,27 @@ namespace Microsoft.ML.Runtime.Data
/// </summary>
public sealed class BootstrapSampleTransform : FilterBase
{
private static class Defaults
{
public const bool Complement = false;
public const bool ShuffleInput = true;
public const int PoolSize = 1000;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.AtMostOnce, HelpText = "Whether this is the out-of-bag sample, that is, all those rows that are not selected by the transform.",
ShortName = "comp")]
public bool Complement;
public bool Complement = Defaults.Complement;

[Argument(ArgumentType.AtMostOnce, HelpText = "The random seed. If unspecified random state will be instead derived from the environment.")]
public uint? Seed;

[Argument(ArgumentType.AtMostOnce, HelpText = "Whether we should attempt to shuffle the source data. By default on, but can be turned off for efficiency.", ShortName = "si")]
public bool ShuffleInput = true;
public bool ShuffleInput = Defaults.ShuffleInput;

[Argument(ArgumentType.LastOccurenceWins, HelpText = "When shuffling the output, the number of output rows to keep in that pool. Note that shuffling of output is completely distinct from shuffling of input.", ShortName = "pool")]
public int PoolSize = 1000;
public int PoolSize = Defaults.PoolSize;
}

internal const string Summary = "Approximate bootstrap sampling.";
Expand DownExpand Up@@ -76,6 +83,25 @@ public BootstrapSampleTransform(IHostEnvironment env, Arguments args, IDataView
_poolSize = args.PoolSize;
}

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="complement">Whether this is the out-of-bag sample, that is, all those rows that are not selected by the transform.</param>
/// <param name="seed">The random seed. If unspecified random state will be instead derived from the environment.</param>
/// <param name="shuffleInput">Whether we should attempt to shuffle the source data. By default on, but can be turned off for efficiency.</param>
/// <param name="poolSize">When shuffling the output, the number of output rows to keep in that pool. Note that shuffling of output is completely distinct from shuffling of input.</param>
public BootstrapSampleTransform(IHostEnvironment env,
IDataView input,
bool complement = Defaults.Complement,
uint? seed = null,
bool shuffleInput = Defaults.ShuffleInput,
int poolSize = Defaults.PoolSize)
: this(env, new Arguments() { Complement = complement, Seed = seed, ShuffleInput = shuffleInput, PoolSize = poolSize }, input)
{
}

private BootstrapSampleTransform(IHost host, ModelLoadContext ctx, IDataView input)
: base(host, input)
{
Expand Down
51 changes: 46 additions & 5 deletions src/Microsoft.ML.Transforms/CategoricalHashTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -86,6 +86,15 @@ public bool TryUnparse(StringBuilder sb)
}
}

private static class Defaults
{
public const int HashBits = 16;
public const uint Seed = 314489979;
public const bool Ordered = true;
public const int InvertHash = 0;
public const CategoricalTransform.OutputKind OutputKind = CategoricalTransform.OutputKind.Bag;
}

/// <summary>
/// This class is a merger of <see cref="HashTransform.Arguments"/> and <see cref="KeyToVectorTransform.Arguments"/>
/// with join option removed
Expand All@@ -97,29 +106,61 @@ public sealed class Arguments : TransformInputBase

[Argument(ArgumentType.AtMostOnce, HelpText = "Number of bits to hash into. Must be between 1 and 30, inclusive.",
ShortName = "bits", SortOrder = 2)]
public int HashBits = 16;
public int HashBits = Defaults.HashBits;

[Argument(ArgumentType.AtMostOnce, HelpText = "Hashing seed")]
public uint Seed = 314489979;
public uint Seed = Defaults.Seed;

[Argument(ArgumentType.AtMostOnce, HelpText = "Whether the position of each term should be included in the hash", ShortName = "ord")]
public bool Ordered = true;
public bool Ordered = Defaults.Ordered;

[Argument(ArgumentType.AtMostOnce,
HelpText = "Limit the number of keys used to generate the slot name to this many. 0 means no invert hashing, -1 means no limit.",
ShortName = "ih")]
public int InvertHash;
public int InvertHash = Defaults.InvertHash;

[Argument(ArgumentType.AtMostOnce, HelpText = "Output kind: Bag (multi-set vector), Ind (indicator vector), or Key (index)",
ShortName = "kind", SortOrder = 102)]
public CategoricalTransform.OutputKind OutputKind = CategoricalTransform.OutputKind.Bag;
public CategoricalTransform.OutputKind OutputKind = Defaults.OutputKind;
}

internal const string Summary = "Converts the categorical value into an indicator array by hashing the value and using the hash as an index in the "
+ "bag. If the input column is a vector, a single indicator bag is returned for it.";

public const string UserName = "Categorical Hash Transform";

/// <summary>
/// A helper method to create <see cref="CategoricalHashTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
/// <param name="hashBits">Number of bits to hash into. Must be between 1 and 30, inclusive.</param>
/// <param name="invertHash">Limit the number of keys used to generate the slot name to this many. 0 means no invert hashing, -1 means no limit.</param>
/// <param name="outputKind">The type of output expected.</param>
public static IDataTransform Create(IHostEnvironment env,
IDataView input,
string name,
string source =null,
int hashBits = Defaults.HashBits,
int invertHash = Defaults.InvertHash,
CategoricalTransform.OutputKind outputKind = Defaults.OutputKind)
{
var args = new Arguments()
{
Column = new[] { new Column(){
Source = source ?? name,
Name = name
}
},
HashBits = hashBits,
InvertHash = invertHash,
OutputKind = outputKind
};
return Create(env, args, input);
}

public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
Expand Down
41 changes: 40 additions & 1 deletion src/Microsoft.ML.Transforms/CategoricalTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,15 +38,27 @@ public static class CategoricalTransform
{
public enum OutputKind : byte
{
/// <summary>
/// Output is a bag (multi-set) vector
/// </summary>
[TGUI(Label = "Output is a bag (multi-set) vector")]
Bag = 1,

/// <summary>
/// Output is an indicator vector
/// </summary>
[TGUI(Label = "Output is an indicator vector")]
Ind = 2,

/// <summary>
/// Output is a key value
/// </summary>
[TGUI(Label = "Output is a key value")]
Key = 3,

/// <summary>
/// Output is binary encoded
/// </summary>
[TGUI(Label = "Output is binary encoded")]
Bin = 4,
}
Expand DownExpand Up@@ -96,14 +108,19 @@ public bool TryUnparse(StringBuilder sb)
}
}

private static class Defaults
{
public const OutputKind OutKind = OutputKind.Ind;
}

public sealed class Arguments : TermTransform.ArgumentsBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)]
public Column[] Column;

[Argument(ArgumentType.AtMostOnce, HelpText = "Output kind: Bag (multi-set vector), Ind (indicator vector), or Key (index)",
ShortName = "kind", SortOrder = 102)]
public OutputKind OutputKind = OutputKind.Ind;
public OutputKind OutputKind = Defaults.OutKind;

public Arguments()
{
Expand All@@ -118,6 +135,28 @@ public Arguments()

public const string UserName = "Categorical Transform";

/// <summary>
/// A helper method to create <see cref="CategoricalTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
/// <param name="outputKind">The type of output expected.</param>
public static IDataTransform Create(IHostEnvironment env, IDataView input, string name, string source = null, OutputKind outputKind = Defaults.OutKind)
{
var args = new Arguments()
{
Column = new[] { new Column(){
Source = source ?? name,
Name = name
}
},
OutputKind = outputKind
};
return Create(env, args, input);
}

public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
Expand Down
25 changes: 24 additions & 1 deletion src/Microsoft.ML.Transforms/CountFeatureSelection.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,17 +28,40 @@ public static class CountFeatureSelectionTransform
public const string Summary = "Selects the slots for which the count of non-default values is greater than or equal to a threshold.";
public const string UserName = "Count Feature Selection Transform";

private static class Defaults
{
public const long Count = 1;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Columns to use for feature selection", ShortName = "col", SortOrder = 1)]
public string[] Column;

[Argument(ArgumentType.Required, HelpText = "If the count of non-default values for a slot is greater than or equal to this threshold, the slot is preserved", ShortName = "c", SortOrder = 1)]
public long Count = 1;
public long Count = Defaults.Count;
}

internal static string RegistrationName = "CountFeatureSelectionTransform";

/// <summary>
/// A helper method to create CountFeatureSelection transform for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="count">If the count of non-default values for a slot is greater than or equal to this threshold, the slot is preserved.</param>
/// <param name="columns">Columns to use for feature selection.</param>
/// <returns></returns>
public static IDataTransform Create(IHostEnvironment env, IDataView input, long count = Defaults.Count, params string[] columns)
{
var args = new Arguments()
{
Column = columns,
Count = count
};
return Create(env, args, input);
}

/// <summary>
/// Create method corresponding to SignatureDataTransform.
/// </summary>
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms"); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions src/Microsoft.ML.Data/Transforms/ConcatTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -90,6 +90,19 @@ public bool TryUnparse(StringBuilder sb)

public sealed class Arguments : TransformInputBase
{
public Arguments()
{
}

public Arguments(string name, params string[] source)
{
Column = new[] { new Column()
{
Name = name,
Source = source
}};
}

[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:srcs)", ShortName = "col", SortOrder = 1)]
public Column[] Column;
}
Expand DownExpand Up@@ -527,6 +540,18 @@ private static VersionInfo GetVersionInfo()

public override ISchema Schema => _bindings;

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Input columns to concatenate.</param>
public ConcatTransform(IHostEnvironment env, IDataView input, string name, params string[] source)
: this(env, new Arguments(name, source), input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand Down
12 changes: 12 additions & 0 deletions src/Microsoft.ML.Data/Transforms/CopyColumnsTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,6 +64,18 @@ private static VersionInfo GetVersionInfo()

private const string RegistrationName = "CopyColumns";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be copied.</param>
public CopyColumnsTransform(IHostEnvironment env, IDataView input, string name, string source)
: this(env, new Arguments(){ Column = new[] { new Column() { Source = source, Name = name }}}, input)
{
}

public CopyColumnsTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, null)
{
Expand Down
24 changes: 24 additions & 0 deletions src/Microsoft.ML.Data/Transforms/DropColumnsTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -237,6 +237,17 @@ private static VersionInfo GetVersionInfo()
private const string DropRegistrationName = "DropColumns";
private const string KeepRegistrationName = "KeepColumns";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="columnsToDrop">Name of the columns to be dropped.</param>
public DropColumnsTransform(IHostEnvironment env, IDataView input, params string[] columnsToDrop)
:this(env, new Arguments() { Column = columnsToDrop }, input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand DownExpand Up@@ -383,4 +394,17 @@ public ValueGetter<TValue> GetGetter<TValue>(int col)
}
}
}

public class KeepColumnsTransform
{
/// <summary>
/// A helper method to create <see cref="KeepColumnsTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="columnsToKeep">Name of the columns to be kept. All other columns will be removed.</param>
/// <returns></returns>
public static IDataTransform Create(IHostEnvironment env, IDataView input, params string[] columnsToKeep)
=> new DropColumnsTransform(env, new DropColumnsTransform.KeepArguments() { Column = columnsToKeep }, input);
}
}
19 changes: 18 additions & 1 deletion src/Microsoft.ML.Data/Transforms/NAFilter.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,13 +28,18 @@ namespace Microsoft.ML.Runtime.Data
{
public sealed class NAFilter : FilterBase
{
private static class Defaults
{
public const bool Complement = false;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Column", ShortName = "col", SortOrder = 1)]
public string[] Column;

[Argument(ArgumentType.Multiple, HelpText = "If true, keep only rows that contain NA values, and filter the rest.")]
public bool Complement;
public bool Complement = Defaults.Complement;
}

private sealed class ColInfo
Expand DownExpand Up@@ -72,6 +77,18 @@ private static VersionInfo GetVersionInfo()
private readonly bool _complement;
private const string RegistrationName = "MissingValueFilter";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="complement">If true, keep only rows that contain NA values, and filter the rest.</param>
/// <param name="columns">Name of the columns. Only these columns will be used to filter rows having 'NA' values.</param>
public NAFilter(IHostEnvironment env, IDataView input, bool complement = Defaults.Complement, params string[] columns)
: this(env, new Arguments() { Column = columns, Complement = complement }, input)
{
}

public NAFilter(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, input)
{
Expand Down
32 changes: 29 additions & 3 deletions src/Microsoft.ML.Transforms/BootstrapSampleTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,20 +25,27 @@ namespace Microsoft.ML.Runtime.Data
/// </summary>
public sealed class BootstrapSampleTransform : FilterBase
{
private static class Defaults
{
public const bool Complement = false;
public const bool ShuffleInput = true;
public const int PoolSize = 1000;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.AtMostOnce, HelpText = "Whether this is the out-of-bag sample, that is, all those rows that are not selected by the transform.",
ShortName = "comp")]
public bool Complement;
public bool Complement = Defaults.Complement;

[Argument(ArgumentType.AtMostOnce, HelpText = "The random seed. If unspecified random state will be instead derived from the environment.")]
public uint? Seed;

[Argument(ArgumentType.AtMostOnce, HelpText = "Whether we should attempt to shuffle the source data. By default on, but can be turned off for efficiency.", ShortName = "si")]
public bool ShuffleInput = true;
public bool ShuffleInput = Defaults.ShuffleInput;

[Argument(ArgumentType.LastOccurenceWins, HelpText = "When shuffling the output, the number of output rows to keep in that pool. Note that shuffling of output is completely distinct from shuffling of input.", ShortName = "pool")]
public int PoolSize = 1000;
public int PoolSize = Defaults.PoolSize;
}

internal const string Summary = "Approximate bootstrap sampling.";
Expand DownExpand Up@@ -76,6 +83,25 @@ public BootstrapSampleTransform(IHostEnvironment env, Arguments args, IDataView
_poolSize = args.PoolSize;
}

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="complement">Whether this is the out-of-bag sample, that is, all those rows that are not selected by the transform.</param>
/// <param name="seed">The random seed. If unspecified random state will be instead derived from the environment.</param>
/// <param name="shuffleInput">Whether we should attempt to shuffle the source data. By default on, but can be turned off for efficiency.</param>
/// <param name="poolSize">When shuffling the output, the number of output rows to keep in that pool. Note that shuffling of output is completely distinct from shuffling of input.</param>
public BootstrapSampleTransform(IHostEnvironment env,
IDataView input,
bool complement = Defaults.Complement,
uint? seed = null,
bool shuffleInput = Defaults.ShuffleInput,
int poolSize = Defaults.PoolSize)
: this(env, new Arguments() { Complement = complement, Seed = seed, ShuffleInput = shuffleInput, PoolSize = poolSize }, input)
{
}

private BootstrapSampleTransform(IHost host, ModelLoadContext ctx, IDataView input)
: base(host, input)
{
Expand Down
51 changes: 46 additions & 5 deletions src/Microsoft.ML.Transforms/CategoricalHashTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -86,6 +86,15 @@ public bool TryUnparse(StringBuilder sb)
}
}

private static class Defaults
{
public const int HashBits = 16;
public const uint Seed = 314489979;
public const bool Ordered = true;
public const int InvertHash = 0;
public const CategoricalTransform.OutputKind OutputKind = CategoricalTransform.OutputKind.Bag;
}

/// <summary>
/// This class is a merger of <see cref="HashTransform.Arguments"/> and <see cref="KeyToVectorTransform.Arguments"/>
/// with join option removed
Expand All@@ -97,29 +106,61 @@ public sealed class Arguments : TransformInputBase

[Argument(ArgumentType.AtMostOnce, HelpText = "Number of bits to hash into. Must be between 1 and 30, inclusive.",
ShortName = "bits", SortOrder = 2)]
public int HashBits = 16;
public int HashBits = Defaults.HashBits;

[Argument(ArgumentType.AtMostOnce, HelpText = "Hashing seed")]
public uint Seed = 314489979;
public uint Seed = Defaults.Seed;

[Argument(ArgumentType.AtMostOnce, HelpText = "Whether the position of each term should be included in the hash", ShortName = "ord")]
public bool Ordered = true;
public bool Ordered = Defaults.Ordered;

[Argument(ArgumentType.AtMostOnce,
HelpText = "Limit the number of keys used to generate the slot name to this many. 0 means no invert hashing, -1 means no limit.",
ShortName = "ih")]
public int InvertHash;
public int InvertHash = Defaults.InvertHash;

[Argument(ArgumentType.AtMostOnce, HelpText = "Output kind: Bag (multi-set vector), Ind (indicator vector), or Key (index)",
ShortName = "kind", SortOrder = 102)]
public CategoricalTransform.OutputKind OutputKind = CategoricalTransform.OutputKind.Bag;
public CategoricalTransform.OutputKind OutputKind = Defaults.OutputKind;
}

internal const string Summary = "Converts the categorical value into an indicator array by hashing the value and using the hash as an index in the "
+ "bag. If the input column is a vector, a single indicator bag is returned for it.";

public const string UserName = "Categorical Hash Transform";

/// <summary>
/// A helper method to create <see cref="CategoricalHashTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
/// <param name="hashBits">Number of bits to hash into. Must be between 1 and 30, inclusive.</param>
/// <param name="invertHash">Limit the number of keys used to generate the slot name to this many. 0 means no invert hashing, -1 means no limit.</param>
/// <param name="outputKind">The type of output expected.</param>
public static IDataTransform Create(IHostEnvironment env,
IDataView input,
string name,
string source =null,
int hashBits = Defaults.HashBits,
int invertHash = Defaults.InvertHash,
CategoricalTransform.OutputKind outputKind = Defaults.OutputKind)
{
var args = new Arguments()
{
Column = new[] { new Column(){
Source = source ?? name,
Name = name
}
},
HashBits = hashBits,
InvertHash = invertHash,
OutputKind = outputKind
};
return Create(env, args, input);
}

public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
Expand Down
41 changes: 40 additions & 1 deletion src/Microsoft.ML.Transforms/CategoricalTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,15 +38,27 @@ public static class CategoricalTransform
{
public enum OutputKind : byte
{
/// <summary>
/// Output is a bag (multi-set) vector
/// </summary>
[TGUI(Label = "Output is a bag (multi-set) vector")]
Bag = 1,

/// <summary>
/// Output is an indicator vector
/// </summary>
[TGUI(Label = "Output is an indicator vector")]
Ind = 2,

/// <summary>
/// Output is a key value
/// </summary>
[TGUI(Label = "Output is a key value")]
Key = 3,

/// <summary>
/// Output is binary encoded
/// </summary>
[TGUI(Label = "Output is binary encoded")]
Bin = 4,
}
Expand DownExpand Up@@ -96,14 +108,19 @@ public bool TryUnparse(StringBuilder sb)
}
}

private static class Defaults
{
public const OutputKind OutKind = OutputKind.Ind;
}

public sealed class Arguments : TermTransform.ArgumentsBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)]
public Column[] Column;

[Argument(ArgumentType.AtMostOnce, HelpText = "Output kind: Bag (multi-set vector), Ind (indicator vector), or Key (index)",
ShortName = "kind", SortOrder = 102)]
public OutputKind OutputKind = OutputKind.Ind;
public OutputKind OutputKind = Defaults.OutKind;

public Arguments()
{
Expand All@@ -118,6 +135,28 @@ public Arguments()

public const string UserName = "Categorical Transform";

/// <summary>
/// A helper method to create <see cref="CategoricalTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
/// <param name="outputKind">The type of output expected.</param>
public static IDataTransform Create(IHostEnvironment env, IDataView input, string name, string source = null, OutputKind outputKind = Defaults.OutKind)
{
var args = new Arguments()
{
Column = new[] { new Column(){
Source = source ?? name,
Name = name
}
},
OutputKind = outputKind
};
return Create(env, args, input);
}

public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
Expand Down
25 changes: 24 additions & 1 deletion src/Microsoft.ML.Transforms/CountFeatureSelection.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,17 +28,40 @@ public static class CountFeatureSelectionTransform
public const string Summary = "Selects the slots for which the count of non-default values is greater than or equal to a threshold.";
public const string UserName = "Count Feature Selection Transform";

private static class Defaults
{
public const long Count = 1;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Columns to use for feature selection", ShortName = "col", SortOrder = 1)]
public string[] Column;

[Argument(ArgumentType.Required, HelpText = "If the count of non-default values for a slot is greater than or equal to this threshold, the slot is preserved", ShortName = "c", SortOrder = 1)]
public long Count = 1;
public long Count = Defaults.Count;
}

internal static string RegistrationName = "CountFeatureSelectionTransform";

/// <summary>
/// A helper method to create CountFeatureSelection transform for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="count">If the count of non-default values for a slot is greater than or equal to this threshold, the slot is preserved.</param>
/// <param name="columns">Columns to use for feature selection.</param>
/// <returns></returns>
public static IDataTransform Create(IHostEnvironment env, IDataView input, long count = Defaults.Count, params string[] columns)
{
var args = new Arguments()
{
Column = columns,
Count = count
};
return Create(env, args, input);
}

/// <summary>
/// Create method corresponding to SignatureDataTransform.
/// </summary>
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links"); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + '
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions src/Microsoft.ML.Data/Transforms/ConcatTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -90,6 +90,19 @@ public bool TryUnparse(StringBuilder sb)

public sealed class Arguments : TransformInputBase
{
public Arguments()
{
}

public Arguments(string name, params string[] source)
{
Column = new[] { new Column()
{
Name = name,
Source = source
}};
}

[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:srcs)", ShortName = "col", SortOrder = 1)]
public Column[] Column;
}
Expand DownExpand Up@@ -527,6 +540,18 @@ private static VersionInfo GetVersionInfo()

public override ISchema Schema => _bindings;

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Input columns to concatenate.</param>
public ConcatTransform(IHostEnvironment env, IDataView input, string name, params string[] source)
: this(env, new Arguments(name, source), input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand Down
12 changes: 12 additions & 0 deletions src/Microsoft.ML.Data/Transforms/CopyColumnsTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,6 +64,18 @@ private static VersionInfo GetVersionInfo()

private const string RegistrationName = "CopyColumns";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be copied.</param>
public CopyColumnsTransform(IHostEnvironment env, IDataView input, string name, string source)
: this(env, new Arguments(){ Column = new[] { new Column() { Source = source, Name = name }}}, input)
{
}

public CopyColumnsTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, null)
{
Expand Down
24 changes: 24 additions & 0 deletions src/Microsoft.ML.Data/Transforms/DropColumnsTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -237,6 +237,17 @@ private static VersionInfo GetVersionInfo()
private const string DropRegistrationName = "DropColumns";
private const string KeepRegistrationName = "KeepColumns";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="columnsToDrop">Name of the columns to be dropped.</param>
public DropColumnsTransform(IHostEnvironment env, IDataView input, params string[] columnsToDrop)
:this(env, new Arguments() { Column = columnsToDrop }, input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand DownExpand Up@@ -383,4 +394,17 @@ public ValueGetter<TValue> GetGetter<TValue>(int col)
}
}
}

public class KeepColumnsTransform
{
/// <summary>
/// A helper method to create <see cref="KeepColumnsTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="columnsToKeep">Name of the columns to be kept. All other columns will be removed.</param>
/// <returns></returns>
public static IDataTransform Create(IHostEnvironment env, IDataView input, params string[] columnsToKeep)
=> new DropColumnsTransform(env, new DropColumnsTransform.KeepArguments() { Column = columnsToKeep }, input);
}
}
19 changes: 18 additions & 1 deletion src/Microsoft.ML.Data/Transforms/NAFilter.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,13 +28,18 @@ namespace Microsoft.ML.Runtime.Data
{
public sealed class NAFilter : FilterBase
{
private static class Defaults
{
public const bool Complement = false;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Column", ShortName = "col", SortOrder = 1)]
public string[] Column;

[Argument(ArgumentType.Multiple, HelpText = "If true, keep only rows that contain NA values, and filter the rest.")]
public bool Complement;
public bool Complement = Defaults.Complement;
}

private sealed class ColInfo
Expand DownExpand Up@@ -72,6 +77,18 @@ private static VersionInfo GetVersionInfo()
private readonly bool _complement;
private const string RegistrationName = "MissingValueFilter";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="complement">If true, keep only rows that contain NA values, and filter the rest.</param>
/// <param name="columns">Name of the columns. Only these columns will be used to filter rows having 'NA' values.</param>
public NAFilter(IHostEnvironment env, IDataView input, bool complement = Defaults.Complement, params string[] columns)
: this(env, new Arguments() { Column = columns, Complement = complement }, input)
{
}

public NAFilter(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, input)
{
Expand Down
32 changes: 29 additions & 3 deletions src/Microsoft.ML.Transforms/BootstrapSampleTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,20 +25,27 @@ namespace Microsoft.ML.Runtime.Data
/// </summary>
public sealed class BootstrapSampleTransform : FilterBase
{
private static class Defaults
{
public const bool Complement = false;
public const bool ShuffleInput = true;
public const int PoolSize = 1000;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.AtMostOnce, HelpText = "Whether this is the out-of-bag sample, that is, all those rows that are not selected by the transform.",
ShortName = "comp")]
public bool Complement;
public bool Complement = Defaults.Complement;

[Argument(ArgumentType.AtMostOnce, HelpText = "The random seed. If unspecified random state will be instead derived from the environment.")]
public uint? Seed;

[Argument(ArgumentType.AtMostOnce, HelpText = "Whether we should attempt to shuffle the source data. By default on, but can be turned off for efficiency.", ShortName = "si")]
public bool ShuffleInput = true;
public bool ShuffleInput = Defaults.ShuffleInput;

[Argument(ArgumentType.LastOccurenceWins, HelpText = "When shuffling the output, the number of output rows to keep in that pool. Note that shuffling of output is completely distinct from shuffling of input.", ShortName = "pool")]
public int PoolSize = 1000;
public int PoolSize = Defaults.PoolSize;
}

internal const string Summary = "Approximate bootstrap sampling.";
Expand DownExpand Up@@ -76,6 +83,25 @@ public BootstrapSampleTransform(IHostEnvironment env, Arguments args, IDataView
_poolSize = args.PoolSize;
}

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="complement">Whether this is the out-of-bag sample, that is, all those rows that are not selected by the transform.</param>
/// <param name="seed">The random seed. If unspecified random state will be instead derived from the environment.</param>
/// <param name="shuffleInput">Whether we should attempt to shuffle the source data. By default on, but can be turned off for efficiency.</param>
/// <param name="poolSize">When shuffling the output, the number of output rows to keep in that pool. Note that shuffling of output is completely distinct from shuffling of input.</param>
public BootstrapSampleTransform(IHostEnvironment env,
IDataView input,
bool complement = Defaults.Complement,
uint? seed = null,
bool shuffleInput = Defaults.ShuffleInput,
int poolSize = Defaults.PoolSize)
: this(env, new Arguments() { Complement = complement, Seed = seed, ShuffleInput = shuffleInput, PoolSize = poolSize }, input)
{
}

private BootstrapSampleTransform(IHost host, ModelLoadContext ctx, IDataView input)
: base(host, input)
{
Expand Down
51 changes: 46 additions & 5 deletions src/Microsoft.ML.Transforms/CategoricalHashTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -86,6 +86,15 @@ public bool TryUnparse(StringBuilder sb)
}
}

private static class Defaults
{
public const int HashBits = 16;
public const uint Seed = 314489979;
public const bool Ordered = true;
public const int InvertHash = 0;
public const CategoricalTransform.OutputKind OutputKind = CategoricalTransform.OutputKind.Bag;
}

/// <summary>
/// This class is a merger of <see cref="HashTransform.Arguments"/> and <see cref="KeyToVectorTransform.Arguments"/>
/// with join option removed
Expand All@@ -97,29 +106,61 @@ public sealed class Arguments : TransformInputBase

[Argument(ArgumentType.AtMostOnce, HelpText = "Number of bits to hash into. Must be between 1 and 30, inclusive.",
ShortName = "bits", SortOrder = 2)]
public int HashBits = 16;
public int HashBits = Defaults.HashBits;

[Argument(ArgumentType.AtMostOnce, HelpText = "Hashing seed")]
public uint Seed = 314489979;
public uint Seed = Defaults.Seed;

[Argument(ArgumentType.AtMostOnce, HelpText = "Whether the position of each term should be included in the hash", ShortName = "ord")]
public bool Ordered = true;
public bool Ordered = Defaults.Ordered;

[Argument(ArgumentType.AtMostOnce,
HelpText = "Limit the number of keys used to generate the slot name to this many. 0 means no invert hashing, -1 means no limit.",
ShortName = "ih")]
public int InvertHash;
public int InvertHash = Defaults.InvertHash;

[Argument(ArgumentType.AtMostOnce, HelpText = "Output kind: Bag (multi-set vector), Ind (indicator vector), or Key (index)",
ShortName = "kind", SortOrder = 102)]
public CategoricalTransform.OutputKind OutputKind = CategoricalTransform.OutputKind.Bag;
public CategoricalTransform.OutputKind OutputKind = Defaults.OutputKind;
}

internal const string Summary = "Converts the categorical value into an indicator array by hashing the value and using the hash as an index in the "
+ "bag. If the input column is a vector, a single indicator bag is returned for it.";

public const string UserName = "Categorical Hash Transform";

/// <summary>
/// A helper method to create <see cref="CategoricalHashTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
/// <param name="hashBits">Number of bits to hash into. Must be between 1 and 30, inclusive.</param>
/// <param name="invertHash">Limit the number of keys used to generate the slot name to this many. 0 means no invert hashing, -1 means no limit.</param>
/// <param name="outputKind">The type of output expected.</param>
public static IDataTransform Create(IHostEnvironment env,
IDataView input,
string name,
string source =null,
int hashBits = Defaults.HashBits,
int invertHash = Defaults.InvertHash,
CategoricalTransform.OutputKind outputKind = Defaults.OutputKind)
{
var args = new Arguments()
{
Column = new[] { new Column(){
Source = source ?? name,
Name = name
}
},
HashBits = hashBits,
InvertHash = invertHash,
OutputKind = outputKind
};
return Create(env, args, input);
}

public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
Expand Down
41 changes: 40 additions & 1 deletion src/Microsoft.ML.Transforms/CategoricalTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,15 +38,27 @@ public static class CategoricalTransform
{
public enum OutputKind : byte
{
/// <summary>
/// Output is a bag (multi-set) vector
/// </summary>
[TGUI(Label = "Output is a bag (multi-set) vector")]
Bag = 1,

/// <summary>
/// Output is an indicator vector
/// </summary>
[TGUI(Label = "Output is an indicator vector")]
Ind = 2,

/// <summary>
/// Output is a key value
/// </summary>
[TGUI(Label = "Output is a key value")]
Key = 3,

/// <summary>
/// Output is binary encoded
/// </summary>
[TGUI(Label = "Output is binary encoded")]
Bin = 4,
}
Expand DownExpand Up@@ -96,14 +108,19 @@ public bool TryUnparse(StringBuilder sb)
}
}

private static class Defaults
{
public const OutputKind OutKind = OutputKind.Ind;
}

public sealed class Arguments : TermTransform.ArgumentsBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)]
public Column[] Column;

[Argument(ArgumentType.AtMostOnce, HelpText = "Output kind: Bag (multi-set vector), Ind (indicator vector), or Key (index)",
ShortName = "kind", SortOrder = 102)]
public OutputKind OutputKind = OutputKind.Ind;
public OutputKind OutputKind = Defaults.OutKind;

public Arguments()
{
Expand All@@ -118,6 +135,28 @@ public Arguments()

public const string UserName = "Categorical Transform";

/// <summary>
/// A helper method to create <see cref="CategoricalTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
/// <param name="outputKind">The type of output expected.</param>
public static IDataTransform Create(IHostEnvironment env, IDataView input, string name, string source = null, OutputKind outputKind = Defaults.OutKind)
{
var args = new Arguments()
{
Column = new[] { new Column(){
Source = source ?? name,
Name = name
}
},
OutputKind = outputKind
};
return Create(env, args, input);
}

public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
Expand Down
25 changes: 24 additions & 1 deletion src/Microsoft.ML.Transforms/CountFeatureSelection.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,17 +28,40 @@ public static class CountFeatureSelectionTransform
public const string Summary = "Selects the slots for which the count of non-default values is greater than or equal to a threshold.";
public const string UserName = "Count Feature Selection Transform";

private static class Defaults
{
public const long Count = 1;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Columns to use for feature selection", ShortName = "col", SortOrder = 1)]
public string[] Column;

[Argument(ArgumentType.Required, HelpText = "If the count of non-default values for a slot is greater than or equal to this threshold, the slot is preserved", ShortName = "c", SortOrder = 1)]
public long Count = 1;
public long Count = Defaults.Count;
}

internal static string RegistrationName = "CountFeatureSelectionTransform";

/// <summary>
/// A helper method to create CountFeatureSelection transform for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="count">If the count of non-default values for a slot is greater than or equal to this threshold, the slot is preserved.</param>
/// <param name="columns">Columns to use for feature selection.</param>
/// <returns></returns>
public static IDataTransform Create(IHostEnvironment env, IDataView input, long count = Defaults.Count, params string[] columns)
{
var args = new Arguments()
{
Column = columns,
Count = count
};
return Create(env, args, input);
}

/// <summary>
/// Create method corresponding to SignatureDataTransform.
/// </summary>
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default"); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions src/Microsoft.ML.Data/Transforms/ConcatTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -90,6 +90,19 @@ public bool TryUnparse(StringBuilder sb)

public sealed class Arguments : TransformInputBase
{
public Arguments()
{
}

public Arguments(string name, params string[] source)
{
Column = new[] { new Column()
{
Name = name,
Source = source
}};
}

[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:srcs)", ShortName = "col", SortOrder = 1)]
public Column[] Column;
}
Expand DownExpand Up@@ -527,6 +540,18 @@ private static VersionInfo GetVersionInfo()

public override ISchema Schema => _bindings;

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Input columns to concatenate.</param>
public ConcatTransform(IHostEnvironment env, IDataView input, string name, params string[] source)
: this(env, new Arguments(name, source), input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand Down
12 changes: 12 additions & 0 deletions src/Microsoft.ML.Data/Transforms/CopyColumnsTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,6 +64,18 @@ private static VersionInfo GetVersionInfo()

private const string RegistrationName = "CopyColumns";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be copied.</param>
public CopyColumnsTransform(IHostEnvironment env, IDataView input, string name, string source)
: this(env, new Arguments(){ Column = new[] { new Column() { Source = source, Name = name }}}, input)
{
}

public CopyColumnsTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, null)
{
Expand Down
24 changes: 24 additions & 0 deletions src/Microsoft.ML.Data/Transforms/DropColumnsTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -237,6 +237,17 @@ private static VersionInfo GetVersionInfo()
private const string DropRegistrationName = "DropColumns";
private const string KeepRegistrationName = "KeepColumns";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="columnsToDrop">Name of the columns to be dropped.</param>
public DropColumnsTransform(IHostEnvironment env, IDataView input, params string[] columnsToDrop)
:this(env, new Arguments() { Column = columnsToDrop }, input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand DownExpand Up@@ -383,4 +394,17 @@ public ValueGetter<TValue> GetGetter<TValue>(int col)
}
}
}

public class KeepColumnsTransform
{
/// <summary>
/// A helper method to create <see cref="KeepColumnsTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="columnsToKeep">Name of the columns to be kept. All other columns will be removed.</param>
/// <returns></returns>
public static IDataTransform Create(IHostEnvironment env, IDataView input, params string[] columnsToKeep)
=> new DropColumnsTransform(env, new DropColumnsTransform.KeepArguments() { Column = columnsToKeep }, input);
}
}
19 changes: 18 additions & 1 deletion src/Microsoft.ML.Data/Transforms/NAFilter.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,13 +28,18 @@ namespace Microsoft.ML.Runtime.Data
{
public sealed class NAFilter : FilterBase
{
private static class Defaults
{
public const bool Complement = false;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Column", ShortName = "col", SortOrder = 1)]
public string[] Column;

[Argument(ArgumentType.Multiple, HelpText = "If true, keep only rows that contain NA values, and filter the rest.")]
public bool Complement;
public bool Complement = Defaults.Complement;
}

private sealed class ColInfo
Expand DownExpand Up@@ -72,6 +77,18 @@ private static VersionInfo GetVersionInfo()
private readonly bool _complement;
private const string RegistrationName = "MissingValueFilter";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="complement">If true, keep only rows that contain NA values, and filter the rest.</param>
/// <param name="columns">Name of the columns. Only these columns will be used to filter rows having 'NA' values.</param>
public NAFilter(IHostEnvironment env, IDataView input, bool complement = Defaults.Complement, params string[] columns)
: this(env, new Arguments() { Column = columns, Complement = complement }, input)
{
}

public NAFilter(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, input)
{
Expand Down
32 changes: 29 additions & 3 deletions src/Microsoft.ML.Transforms/BootstrapSampleTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,20 +25,27 @@ namespace Microsoft.ML.Runtime.Data
/// </summary>
public sealed class BootstrapSampleTransform : FilterBase
{
private static class Defaults
{
public const bool Complement = false;
public const bool ShuffleInput = true;
public const int PoolSize = 1000;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.AtMostOnce, HelpText = "Whether this is the out-of-bag sample, that is, all those rows that are not selected by the transform.",
ShortName = "comp")]
public bool Complement;
public bool Complement = Defaults.Complement;

[Argument(ArgumentType.AtMostOnce, HelpText = "The random seed. If unspecified random state will be instead derived from the environment.")]
public uint? Seed;

[Argument(ArgumentType.AtMostOnce, HelpText = "Whether we should attempt to shuffle the source data. By default on, but can be turned off for efficiency.", ShortName = "si")]
public bool ShuffleInput = true;
public bool ShuffleInput = Defaults.ShuffleInput;

[Argument(ArgumentType.LastOccurenceWins, HelpText = "When shuffling the output, the number of output rows to keep in that pool. Note that shuffling of output is completely distinct from shuffling of input.", ShortName = "pool")]
public int PoolSize = 1000;
public int PoolSize = Defaults.PoolSize;
}

internal const string Summary = "Approximate bootstrap sampling.";
Expand DownExpand Up@@ -76,6 +83,25 @@ public BootstrapSampleTransform(IHostEnvironment env, Arguments args, IDataView
_poolSize = args.PoolSize;
}

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="complement">Whether this is the out-of-bag sample, that is, all those rows that are not selected by the transform.</param>
/// <param name="seed">The random seed. If unspecified random state will be instead derived from the environment.</param>
/// <param name="shuffleInput">Whether we should attempt to shuffle the source data. By default on, but can be turned off for efficiency.</param>
/// <param name="poolSize">When shuffling the output, the number of output rows to keep in that pool. Note that shuffling of output is completely distinct from shuffling of input.</param>
public BootstrapSampleTransform(IHostEnvironment env,
IDataView input,
bool complement = Defaults.Complement,
uint? seed = null,
bool shuffleInput = Defaults.ShuffleInput,
int poolSize = Defaults.PoolSize)
: this(env, new Arguments() { Complement = complement, Seed = seed, ShuffleInput = shuffleInput, PoolSize = poolSize }, input)
{
}

private BootstrapSampleTransform(IHost host, ModelLoadContext ctx, IDataView input)
: base(host, input)
{
Expand Down
51 changes: 46 additions & 5 deletions src/Microsoft.ML.Transforms/CategoricalHashTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -86,6 +86,15 @@ public bool TryUnparse(StringBuilder sb)
}
}

private static class Defaults
{
public const int HashBits = 16;
public const uint Seed = 314489979;
public const bool Ordered = true;
public const int InvertHash = 0;
public const CategoricalTransform.OutputKind OutputKind = CategoricalTransform.OutputKind.Bag;
}

/// <summary>
/// This class is a merger of <see cref="HashTransform.Arguments"/> and <see cref="KeyToVectorTransform.Arguments"/>
/// with join option removed
Expand All@@ -97,29 +106,61 @@ public sealed class Arguments : TransformInputBase

[Argument(ArgumentType.AtMostOnce, HelpText = "Number of bits to hash into. Must be between 1 and 30, inclusive.",
ShortName = "bits", SortOrder = 2)]
public int HashBits = 16;
public int HashBits = Defaults.HashBits;

[Argument(ArgumentType.AtMostOnce, HelpText = "Hashing seed")]
public uint Seed = 314489979;
public uint Seed = Defaults.Seed;

[Argument(ArgumentType.AtMostOnce, HelpText = "Whether the position of each term should be included in the hash", ShortName = "ord")]
public bool Ordered = true;
public bool Ordered = Defaults.Ordered;

[Argument(ArgumentType.AtMostOnce,
HelpText = "Limit the number of keys used to generate the slot name to this many. 0 means no invert hashing, -1 means no limit.",
ShortName = "ih")]
public int InvertHash;
public int InvertHash = Defaults.InvertHash;

[Argument(ArgumentType.AtMostOnce, HelpText = "Output kind: Bag (multi-set vector), Ind (indicator vector), or Key (index)",
ShortName = "kind", SortOrder = 102)]
public CategoricalTransform.OutputKind OutputKind = CategoricalTransform.OutputKind.Bag;
public CategoricalTransform.OutputKind OutputKind = Defaults.OutputKind;
}

internal const string Summary = "Converts the categorical value into an indicator array by hashing the value and using the hash as an index in the "
+ "bag. If the input column is a vector, a single indicator bag is returned for it.";

public const string UserName = "Categorical Hash Transform";

/// <summary>
/// A helper method to create <see cref="CategoricalHashTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
/// <param name="hashBits">Number of bits to hash into. Must be between 1 and 30, inclusive.</param>
/// <param name="invertHash">Limit the number of keys used to generate the slot name to this many. 0 means no invert hashing, -1 means no limit.</param>
/// <param name="outputKind">The type of output expected.</param>
public static IDataTransform Create(IHostEnvironment env,
IDataView input,
string name,
string source =null,
int hashBits = Defaults.HashBits,
int invertHash = Defaults.InvertHash,
CategoricalTransform.OutputKind outputKind = Defaults.OutputKind)
{
var args = new Arguments()
{
Column = new[] { new Column(){
Source = source ?? name,
Name = name
}
},
HashBits = hashBits,
InvertHash = invertHash,
OutputKind = outputKind
};
return Create(env, args, input);
}

public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
Expand Down
41 changes: 40 additions & 1 deletion src/Microsoft.ML.Transforms/CategoricalTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,15 +38,27 @@ public static class CategoricalTransform
{
public enum OutputKind : byte
{
/// <summary>
/// Output is a bag (multi-set) vector
/// </summary>
[TGUI(Label = "Output is a bag (multi-set) vector")]
Bag = 1,

/// <summary>
/// Output is an indicator vector
/// </summary>
[TGUI(Label = "Output is an indicator vector")]
Ind = 2,

/// <summary>
/// Output is a key value
/// </summary>
[TGUI(Label = "Output is a key value")]
Key = 3,

/// <summary>
/// Output is binary encoded
/// </summary>
[TGUI(Label = "Output is binary encoded")]
Bin = 4,
}
Expand DownExpand Up@@ -96,14 +108,19 @@ public bool TryUnparse(StringBuilder sb)
}
}

private static class Defaults
{
public const OutputKind OutKind = OutputKind.Ind;
}

public sealed class Arguments : TermTransform.ArgumentsBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)]
public Column[] Column;

[Argument(ArgumentType.AtMostOnce, HelpText = "Output kind: Bag (multi-set vector), Ind (indicator vector), or Key (index)",
ShortName = "kind", SortOrder = 102)]
public OutputKind OutputKind = OutputKind.Ind;
public OutputKind OutputKind = Defaults.OutKind;

public Arguments()
{
Expand All@@ -118,6 +135,28 @@ public Arguments()

public const string UserName = "Categorical Transform";

/// <summary>
/// A helper method to create <see cref="CategoricalTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
/// <param name="outputKind">The type of output expected.</param>
public static IDataTransform Create(IHostEnvironment env, IDataView input, string name, string source = null, OutputKind outputKind = Defaults.OutKind)
{
var args = new Arguments()
{
Column = new[] { new Column(){
Source = source ?? name,
Name = name
}
},
OutputKind = outputKind
};
return Create(env, args, input);
}

public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
Expand Down
25 changes: 24 additions & 1 deletion src/Microsoft.ML.Transforms/CountFeatureSelection.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,17 +28,40 @@ public static class CountFeatureSelectionTransform
public const string Summary = "Selects the slots for which the count of non-default values is greater than or equal to a threshold.";
public const string UserName = "Count Feature Selection Transform";

private static class Defaults
{
public const long Count = 1;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Columns to use for feature selection", ShortName = "col", SortOrder = 1)]
public string[] Column;

[Argument(ArgumentType.Required, HelpText = "If the count of non-default values for a slot is greater than or equal to this threshold, the slot is preserved", ShortName = "c", SortOrder = 1)]
public long Count = 1;
public long Count = Defaults.Count;
}

internal static string RegistrationName = "CountFeatureSelectionTransform";

/// <summary>
/// A helper method to create CountFeatureSelection transform for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="count">If the count of non-default values for a slot is greater than or equal to this threshold, the slot is preserved.</param>
/// <param name="columns">Columns to use for feature selection.</param>
/// <returns></returns>
public static IDataTransform Create(IHostEnvironment env, IDataView input, long count = Defaults.Count, params string[] columns)
{
var args = new Arguments()
{
Column = columns,
Count = count
};
return Create(env, args, input);
}

/// <summary>
/// Create method corresponding to SignatureDataTransform.
/// </summary>
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers"); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions src/Microsoft.ML.Data/Transforms/ConcatTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -90,6 +90,19 @@ public bool TryUnparse(StringBuilder sb)

public sealed class Arguments : TransformInputBase
{
public Arguments()
{
}

public Arguments(string name, params string[] source)
{
Column = new[] { new Column()
{
Name = name,
Source = source
}};
}

[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:srcs)", ShortName = "col", SortOrder = 1)]
public Column[] Column;
}
Expand DownExpand Up@@ -527,6 +540,18 @@ private static VersionInfo GetVersionInfo()

public override ISchema Schema => _bindings;

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Input columns to concatenate.</param>
public ConcatTransform(IHostEnvironment env, IDataView input, string name, params string[] source)
: this(env, new Arguments(name, source), input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand Down
12 changes: 12 additions & 0 deletions src/Microsoft.ML.Data/Transforms/CopyColumnsTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,6 +64,18 @@ private static VersionInfo GetVersionInfo()

private const string RegistrationName = "CopyColumns";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be copied.</param>
public CopyColumnsTransform(IHostEnvironment env, IDataView input, string name, string source)
: this(env, new Arguments(){ Column = new[] { new Column() { Source = source, Name = name }}}, input)
{
}

public CopyColumnsTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, null)
{
Expand Down
24 changes: 24 additions & 0 deletions src/Microsoft.ML.Data/Transforms/DropColumnsTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -237,6 +237,17 @@ private static VersionInfo GetVersionInfo()
private const string DropRegistrationName = "DropColumns";
private const string KeepRegistrationName = "KeepColumns";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="columnsToDrop">Name of the columns to be dropped.</param>
public DropColumnsTransform(IHostEnvironment env, IDataView input, params string[] columnsToDrop)
:this(env, new Arguments() { Column = columnsToDrop }, input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand DownExpand Up@@ -383,4 +394,17 @@ public ValueGetter<TValue> GetGetter<TValue>(int col)
}
}
}

public class KeepColumnsTransform
{
/// <summary>
/// A helper method to create <see cref="KeepColumnsTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="columnsToKeep">Name of the columns to be kept. All other columns will be removed.</param>
/// <returns></returns>
public static IDataTransform Create(IHostEnvironment env, IDataView input, params string[] columnsToKeep)
=> new DropColumnsTransform(env, new DropColumnsTransform.KeepArguments() { Column = columnsToKeep }, input);
}
}
19 changes: 18 additions & 1 deletion src/Microsoft.ML.Data/Transforms/NAFilter.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,13 +28,18 @@ namespace Microsoft.ML.Runtime.Data
{
public sealed class NAFilter : FilterBase
{
private static class Defaults
{
public const bool Complement = false;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Column", ShortName = "col", SortOrder = 1)]
public string[] Column;

[Argument(ArgumentType.Multiple, HelpText = "If true, keep only rows that contain NA values, and filter the rest.")]
public bool Complement;
public bool Complement = Defaults.Complement;
}

private sealed class ColInfo
Expand DownExpand Up@@ -72,6 +77,18 @@ private static VersionInfo GetVersionInfo()
private readonly bool _complement;
private const string RegistrationName = "MissingValueFilter";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="complement">If true, keep only rows that contain NA values, and filter the rest.</param>
/// <param name="columns">Name of the columns. Only these columns will be used to filter rows having 'NA' values.</param>
public NAFilter(IHostEnvironment env, IDataView input, bool complement = Defaults.Complement, params string[] columns)
: this(env, new Arguments() { Column = columns, Complement = complement }, input)
{
}

public NAFilter(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, input)
{
Expand Down
32 changes: 29 additions & 3 deletions src/Microsoft.ML.Transforms/BootstrapSampleTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,20 +25,27 @@ namespace Microsoft.ML.Runtime.Data
/// </summary>
public sealed class BootstrapSampleTransform : FilterBase
{
private static class Defaults
{
public const bool Complement = false;
public const bool ShuffleInput = true;
public const int PoolSize = 1000;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.AtMostOnce, HelpText = "Whether this is the out-of-bag sample, that is, all those rows that are not selected by the transform.",
ShortName = "comp")]
public bool Complement;
public bool Complement = Defaults.Complement;

[Argument(ArgumentType.AtMostOnce, HelpText = "The random seed. If unspecified random state will be instead derived from the environment.")]
public uint? Seed;

[Argument(ArgumentType.AtMostOnce, HelpText = "Whether we should attempt to shuffle the source data. By default on, but can be turned off for efficiency.", ShortName = "si")]
public bool ShuffleInput = true;
public bool ShuffleInput = Defaults.ShuffleInput;

[Argument(ArgumentType.LastOccurenceWins, HelpText = "When shuffling the output, the number of output rows to keep in that pool. Note that shuffling of output is completely distinct from shuffling of input.", ShortName = "pool")]
public int PoolSize = 1000;
public int PoolSize = Defaults.PoolSize;
}

internal const string Summary = "Approximate bootstrap sampling.";
Expand DownExpand Up@@ -76,6 +83,25 @@ public BootstrapSampleTransform(IHostEnvironment env, Arguments args, IDataView
_poolSize = args.PoolSize;
}

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="complement">Whether this is the out-of-bag sample, that is, all those rows that are not selected by the transform.</param>
/// <param name="seed">The random seed. If unspecified random state will be instead derived from the environment.</param>
/// <param name="shuffleInput">Whether we should attempt to shuffle the source data. By default on, but can be turned off for efficiency.</param>
/// <param name="poolSize">When shuffling the output, the number of output rows to keep in that pool. Note that shuffling of output is completely distinct from shuffling of input.</param>
public BootstrapSampleTransform(IHostEnvironment env,
IDataView input,
bool complement = Defaults.Complement,
uint? seed = null,
bool shuffleInput = Defaults.ShuffleInput,
int poolSize = Defaults.PoolSize)
: this(env, new Arguments() { Complement = complement, Seed = seed, ShuffleInput = shuffleInput, PoolSize = poolSize }, input)
{
}

private BootstrapSampleTransform(IHost host, ModelLoadContext ctx, IDataView input)
: base(host, input)
{
Expand Down
51 changes: 46 additions & 5 deletions src/Microsoft.ML.Transforms/CategoricalHashTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -86,6 +86,15 @@ public bool TryUnparse(StringBuilder sb)
}
}

private static class Defaults
{
public const int HashBits = 16;
public const uint Seed = 314489979;
public const bool Ordered = true;
public const int InvertHash = 0;
public const CategoricalTransform.OutputKind OutputKind = CategoricalTransform.OutputKind.Bag;
}

/// <summary>
/// This class is a merger of <see cref="HashTransform.Arguments"/> and <see cref="KeyToVectorTransform.Arguments"/>
/// with join option removed
Expand All@@ -97,29 +106,61 @@ public sealed class Arguments : TransformInputBase

[Argument(ArgumentType.AtMostOnce, HelpText = "Number of bits to hash into. Must be between 1 and 30, inclusive.",
ShortName = "bits", SortOrder = 2)]
public int HashBits = 16;
public int HashBits = Defaults.HashBits;

[Argument(ArgumentType.AtMostOnce, HelpText = "Hashing seed")]
public uint Seed = 314489979;
public uint Seed = Defaults.Seed;

[Argument(ArgumentType.AtMostOnce, HelpText = "Whether the position of each term should be included in the hash", ShortName = "ord")]
public bool Ordered = true;
public bool Ordered = Defaults.Ordered;

[Argument(ArgumentType.AtMostOnce,
HelpText = "Limit the number of keys used to generate the slot name to this many. 0 means no invert hashing, -1 means no limit.",
ShortName = "ih")]
public int InvertHash;
public int InvertHash = Defaults.InvertHash;

[Argument(ArgumentType.AtMostOnce, HelpText = "Output kind: Bag (multi-set vector), Ind (indicator vector), or Key (index)",
ShortName = "kind", SortOrder = 102)]
public CategoricalTransform.OutputKind OutputKind = CategoricalTransform.OutputKind.Bag;
public CategoricalTransform.OutputKind OutputKind = Defaults.OutputKind;
}

internal const string Summary = "Converts the categorical value into an indicator array by hashing the value and using the hash as an index in the "
+ "bag. If the input column is a vector, a single indicator bag is returned for it.";

public const string UserName = "Categorical Hash Transform";

/// <summary>
/// A helper method to create <see cref="CategoricalHashTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
/// <param name="hashBits">Number of bits to hash into. Must be between 1 and 30, inclusive.</param>
/// <param name="invertHash">Limit the number of keys used to generate the slot name to this many. 0 means no invert hashing, -1 means no limit.</param>
/// <param name="outputKind">The type of output expected.</param>
public static IDataTransform Create(IHostEnvironment env,
IDataView input,
string name,
string source =null,
int hashBits = Defaults.HashBits,
int invertHash = Defaults.InvertHash,
CategoricalTransform.OutputKind outputKind = Defaults.OutputKind)
{
var args = new Arguments()
{
Column = new[] { new Column(){
Source = source ?? name,
Name = name
}
},
HashBits = hashBits,
InvertHash = invertHash,
OutputKind = outputKind
};
return Create(env, args, input);
}

public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
Expand Down
41 changes: 40 additions & 1 deletion src/Microsoft.ML.Transforms/CategoricalTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,15 +38,27 @@ public static class CategoricalTransform
{
public enum OutputKind : byte
{
/// <summary>
/// Output is a bag (multi-set) vector
/// </summary>
[TGUI(Label = "Output is a bag (multi-set) vector")]
Bag = 1,

/// <summary>
/// Output is an indicator vector
/// </summary>
[TGUI(Label = "Output is an indicator vector")]
Ind = 2,

/// <summary>
/// Output is a key value
/// </summary>
[TGUI(Label = "Output is a key value")]
Key = 3,

/// <summary>
/// Output is binary encoded
/// </summary>
[TGUI(Label = "Output is binary encoded")]
Bin = 4,
}
Expand DownExpand Up@@ -96,14 +108,19 @@ public bool TryUnparse(StringBuilder sb)
}
}

private static class Defaults
{
public const OutputKind OutKind = OutputKind.Ind;
}

public sealed class Arguments : TermTransform.ArgumentsBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)]
public Column[] Column;

[Argument(ArgumentType.AtMostOnce, HelpText = "Output kind: Bag (multi-set vector), Ind (indicator vector), or Key (index)",
ShortName = "kind", SortOrder = 102)]
public OutputKind OutputKind = OutputKind.Ind;
public OutputKind OutputKind = Defaults.OutKind;

public Arguments()
{
Expand All@@ -118,6 +135,28 @@ public Arguments()

public const string UserName = "Categorical Transform";

/// <summary>
/// A helper method to create <see cref="CategoricalTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
/// <param name="outputKind">The type of output expected.</param>
public static IDataTransform Create(IHostEnvironment env, IDataView input, string name, string source = null, OutputKind outputKind = Defaults.OutKind)
{
var args = new Arguments()
{
Column = new[] { new Column(){
Source = source ?? name,
Name = name
}
},
OutputKind = outputKind
};
return Create(env, args, input);
}

public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
Expand Down
25 changes: 24 additions & 1 deletion src/Microsoft.ML.Transforms/CountFeatureSelection.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,17 +28,40 @@ public static class CountFeatureSelectionTransform
public const string Summary = "Selects the slots for which the count of non-default values is greater than or equal to a threshold.";
public const string UserName = "Count Feature Selection Transform";

private static class Defaults
{
public const long Count = 1;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Columns to use for feature selection", ShortName = "col", SortOrder = 1)]
public string[] Column;

[Argument(ArgumentType.Required, HelpText = "If the count of non-default values for a slot is greater than or equal to this threshold, the slot is preserved", ShortName = "c", SortOrder = 1)]
public long Count = 1;
public long Count = Defaults.Count;
}

internal static string RegistrationName = "CountFeatureSelectionTransform";

/// <summary>
/// A helper method to create CountFeatureSelection transform for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="count">If the count of non-default values for a slot is greater than or equal to this threshold, the slot is preserved.</param>
/// <param name="columns">Columns to use for feature selection.</param>
/// <returns></returns>
public static IDataTransform Create(IHostEnvironment env, IDataView input, long count = Defaults.Count, params string[] columns)
{
var args = new Arguments()
{
Column = columns,
Count = count
};
return Create(env, args, input);
}

/// <summary>
/// Create method corresponding to SignatureDataTransform.
/// </summary>
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode"); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })();
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions src/Microsoft.ML.Data/Transforms/ConcatTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -90,6 +90,19 @@ public bool TryUnparse(StringBuilder sb)

public sealed class Arguments : TransformInputBase
{
public Arguments()
{
}

public Arguments(string name, params string[] source)
{
Column = new[] { new Column()
{
Name = name,
Source = source
}};
}

[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:srcs)", ShortName = "col", SortOrder = 1)]
public Column[] Column;
}
Expand DownExpand Up@@ -527,6 +540,18 @@ private static VersionInfo GetVersionInfo()

public override ISchema Schema => _bindings;

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Input columns to concatenate.</param>
public ConcatTransform(IHostEnvironment env, IDataView input, string name, params string[] source)
: this(env, new Arguments(name, source), input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand Down
12 changes: 12 additions & 0 deletions src/Microsoft.ML.Data/Transforms/CopyColumnsTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,6 +64,18 @@ private static VersionInfo GetVersionInfo()

private const string RegistrationName = "CopyColumns";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be copied.</param>
public CopyColumnsTransform(IHostEnvironment env, IDataView input, string name, string source)
: this(env, new Arguments(){ Column = new[] { new Column() { Source = source, Name = name }}}, input)
{
}

public CopyColumnsTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, null)
{
Expand Down
24 changes: 24 additions & 0 deletions src/Microsoft.ML.Data/Transforms/DropColumnsTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -237,6 +237,17 @@ private static VersionInfo GetVersionInfo()
private const string DropRegistrationName = "DropColumns";
private const string KeepRegistrationName = "KeepColumns";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="columnsToDrop">Name of the columns to be dropped.</param>
public DropColumnsTransform(IHostEnvironment env, IDataView input, params string[] columnsToDrop)
:this(env, new Arguments() { Column = columnsToDrop }, input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand DownExpand Up@@ -383,4 +394,17 @@ public ValueGetter<TValue> GetGetter<TValue>(int col)
}
}
}

public class KeepColumnsTransform
{
/// <summary>
/// A helper method to create <see cref="KeepColumnsTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="columnsToKeep">Name of the columns to be kept. All other columns will be removed.</param>
/// <returns></returns>
public static IDataTransform Create(IHostEnvironment env, IDataView input, params string[] columnsToKeep)
=> new DropColumnsTransform(env, new DropColumnsTransform.KeepArguments() { Column = columnsToKeep }, input);
}
}
19 changes: 18 additions & 1 deletion src/Microsoft.ML.Data/Transforms/NAFilter.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,13 +28,18 @@ namespace Microsoft.ML.Runtime.Data
{
public sealed class NAFilter : FilterBase
{
private static class Defaults
{
public const bool Complement = false;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Column", ShortName = "col", SortOrder = 1)]
public string[] Column;

[Argument(ArgumentType.Multiple, HelpText = "If true, keep only rows that contain NA values, and filter the rest.")]
public bool Complement;
public bool Complement = Defaults.Complement;
}

private sealed class ColInfo
Expand DownExpand Up@@ -72,6 +77,18 @@ private static VersionInfo GetVersionInfo()
private readonly bool _complement;
private const string RegistrationName = "MissingValueFilter";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="complement">If true, keep only rows that contain NA values, and filter the rest.</param>
/// <param name="columns">Name of the columns. Only these columns will be used to filter rows having 'NA' values.</param>
public NAFilter(IHostEnvironment env, IDataView input, bool complement = Defaults.Complement, params string[] columns)
: this(env, new Arguments() { Column = columns, Complement = complement }, input)
{
}

public NAFilter(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, input)
{
Expand Down
32 changes: 29 additions & 3 deletions src/Microsoft.ML.Transforms/BootstrapSampleTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,20 +25,27 @@ namespace Microsoft.ML.Runtime.Data
/// </summary>
public sealed class BootstrapSampleTransform : FilterBase
{
private static class Defaults
{
public const bool Complement = false;
public const bool ShuffleInput = true;
public const int PoolSize = 1000;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.AtMostOnce, HelpText = "Whether this is the out-of-bag sample, that is, all those rows that are not selected by the transform.",
ShortName = "comp")]
public bool Complement;
public bool Complement = Defaults.Complement;

[Argument(ArgumentType.AtMostOnce, HelpText = "The random seed. If unspecified random state will be instead derived from the environment.")]
public uint? Seed;

[Argument(ArgumentType.AtMostOnce, HelpText = "Whether we should attempt to shuffle the source data. By default on, but can be turned off for efficiency.", ShortName = "si")]
public bool ShuffleInput = true;
public bool ShuffleInput = Defaults.ShuffleInput;

[Argument(ArgumentType.LastOccurenceWins, HelpText = "When shuffling the output, the number of output rows to keep in that pool. Note that shuffling of output is completely distinct from shuffling of input.", ShortName = "pool")]
public int PoolSize = 1000;
public int PoolSize = Defaults.PoolSize;
}

internal const string Summary = "Approximate bootstrap sampling.";
Expand DownExpand Up@@ -76,6 +83,25 @@ public BootstrapSampleTransform(IHostEnvironment env, Arguments args, IDataView
_poolSize = args.PoolSize;
}

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="complement">Whether this is the out-of-bag sample, that is, all those rows that are not selected by the transform.</param>
/// <param name="seed">The random seed. If unspecified random state will be instead derived from the environment.</param>
/// <param name="shuffleInput">Whether we should attempt to shuffle the source data. By default on, but can be turned off for efficiency.</param>
/// <param name="poolSize">When shuffling the output, the number of output rows to keep in that pool. Note that shuffling of output is completely distinct from shuffling of input.</param>
public BootstrapSampleTransform(IHostEnvironment env,
IDataView input,
bool complement = Defaults.Complement,
uint? seed = null,
bool shuffleInput = Defaults.ShuffleInput,
int poolSize = Defaults.PoolSize)
: this(env, new Arguments() { Complement = complement, Seed = seed, ShuffleInput = shuffleInput, PoolSize = poolSize }, input)
{
}

private BootstrapSampleTransform(IHost host, ModelLoadContext ctx, IDataView input)
: base(host, input)
{
Expand Down
51 changes: 46 additions & 5 deletions src/Microsoft.ML.Transforms/CategoricalHashTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -86,6 +86,15 @@ public bool TryUnparse(StringBuilder sb)
}
}

private static class Defaults
{
public const int HashBits = 16;
public const uint Seed = 314489979;
public const bool Ordered = true;
public const int InvertHash = 0;
public const CategoricalTransform.OutputKind OutputKind = CategoricalTransform.OutputKind.Bag;
}

/// <summary>
/// This class is a merger of <see cref="HashTransform.Arguments"/> and <see cref="KeyToVectorTransform.Arguments"/>
/// with join option removed
Expand All@@ -97,29 +106,61 @@ public sealed class Arguments : TransformInputBase

[Argument(ArgumentType.AtMostOnce, HelpText = "Number of bits to hash into. Must be between 1 and 30, inclusive.",
ShortName = "bits", SortOrder = 2)]
public int HashBits = 16;
public int HashBits = Defaults.HashBits;

[Argument(ArgumentType.AtMostOnce, HelpText = "Hashing seed")]
public uint Seed = 314489979;
public uint Seed = Defaults.Seed;

[Argument(ArgumentType.AtMostOnce, HelpText = "Whether the position of each term should be included in the hash", ShortName = "ord")]
public bool Ordered = true;
public bool Ordered = Defaults.Ordered;

[Argument(ArgumentType.AtMostOnce,
HelpText = "Limit the number of keys used to generate the slot name to this many. 0 means no invert hashing, -1 means no limit.",
ShortName = "ih")]
public int InvertHash;
public int InvertHash = Defaults.InvertHash;

[Argument(ArgumentType.AtMostOnce, HelpText = "Output kind: Bag (multi-set vector), Ind (indicator vector), or Key (index)",
ShortName = "kind", SortOrder = 102)]
public CategoricalTransform.OutputKind OutputKind = CategoricalTransform.OutputKind.Bag;
public CategoricalTransform.OutputKind OutputKind = Defaults.OutputKind;
}

internal const string Summary = "Converts the categorical value into an indicator array by hashing the value and using the hash as an index in the "
+ "bag. If the input column is a vector, a single indicator bag is returned for it.";

public const string UserName = "Categorical Hash Transform";

/// <summary>
/// A helper method to create <see cref="CategoricalHashTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
/// <param name="hashBits">Number of bits to hash into. Must be between 1 and 30, inclusive.</param>
/// <param name="invertHash">Limit the number of keys used to generate the slot name to this many. 0 means no invert hashing, -1 means no limit.</param>
/// <param name="outputKind">The type of output expected.</param>
public static IDataTransform Create(IHostEnvironment env,
IDataView input,
string name,
string source =null,
int hashBits = Defaults.HashBits,
int invertHash = Defaults.InvertHash,
CategoricalTransform.OutputKind outputKind = Defaults.OutputKind)
{
var args = new Arguments()
{
Column = new[] { new Column(){
Source = source ?? name,
Name = name
}
},
HashBits = hashBits,
InvertHash = invertHash,
OutputKind = outputKind
};
return Create(env, args, input);
}

public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
Expand Down
41 changes: 40 additions & 1 deletion src/Microsoft.ML.Transforms/CategoricalTransform.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,15 +38,27 @@ public static class CategoricalTransform
{
public enum OutputKind : byte
{
/// <summary>
/// Output is a bag (multi-set) vector
/// </summary>
[TGUI(Label = "Output is a bag (multi-set) vector")]
Bag = 1,

/// <summary>
/// Output is an indicator vector
/// </summary>
[TGUI(Label = "Output is an indicator vector")]
Ind = 2,

/// <summary>
/// Output is a key value
/// </summary>
[TGUI(Label = "Output is a key value")]
Key = 3,

/// <summary>
/// Output is binary encoded
/// </summary>
[TGUI(Label = "Output is binary encoded")]
Bin = 4,
}
Expand DownExpand Up@@ -96,14 +108,19 @@ public bool TryUnparse(StringBuilder sb)
}
}

private static class Defaults
{
public const OutputKind OutKind = OutputKind.Ind;
}

public sealed class Arguments : TermTransform.ArgumentsBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)]
public Column[] Column;

[Argument(ArgumentType.AtMostOnce, HelpText = "Output kind: Bag (multi-set vector), Ind (indicator vector), or Key (index)",
ShortName = "kind", SortOrder = 102)]
public OutputKind OutputKind = OutputKind.Ind;
public OutputKind OutputKind = Defaults.OutKind;

public Arguments()
{
Expand All@@ -118,6 +135,28 @@ public Arguments()

public const string UserName = "Categorical Transform";

/// <summary>
/// A helper method to create <see cref="CategoricalTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
/// <param name="outputKind">The type of output expected.</param>
public static IDataTransform Create(IHostEnvironment env, IDataView input, string name, string source = null, OutputKind outputKind = Defaults.OutKind)
{
var args = new Arguments()
{
Column = new[] { new Column(){
Source = source ?? name,
Name = name
}
},
OutputKind = outputKind
};
return Create(env, args, input);
}

public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
Expand Down
25 changes: 24 additions & 1 deletion src/Microsoft.ML.Transforms/CountFeatureSelection.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,17 +28,40 @@ public static class CountFeatureSelectionTransform
public const string Summary = "Selects the slots for which the count of non-default values is greater than or equal to a threshold.";
public const string UserName = "Count Feature Selection Transform";

private static class Defaults
{
public const long Count = 1;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Columns to use for feature selection", ShortName = "col", SortOrder = 1)]
public string[] Column;

[Argument(ArgumentType.Required, HelpText = "If the count of non-default values for a slot is greater than or equal to this threshold, the slot is preserved", ShortName = "c", SortOrder = 1)]
public long Count = 1;
public long Count = Defaults.Count;
}

internal static string RegistrationName = "CountFeatureSelectionTransform";

/// <summary>
/// A helper method to create CountFeatureSelection transform for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="count">If the count of non-default values for a slot is greater than or equal to this threshold, the slot is preserved.</param>
/// <param name="columns">Columns to use for feature selection.</param>
/// <returns></returns>
public static IDataTransform Create(IHostEnvironment env, IDataView input, long count = Defaults.Count, params string[] columns)
{
var args = new Arguments()
{
Column = columns,
Count = count
};
return Create(env, args, input);
}

/// <summary>
/// Create method corresponding to SignatureDataTransform.
/// </summary>
Expand Down
Loading