[Part 3] Added convenience constructors for set of transforms.#520

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zeahmed merged 6 commits into from
Jul 16, 2018
Merged

[Part 3] Added convenience constructors for set of transforms. #520

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d9c4a29
[Part 3] Added convenience constructors for set of transforms.
zeahmed Jul 10, 2018
6cd77eb
Addressed reviewers' comments.
zeahmed Jul 12, 2018
75a8b6d
Addressed reviewers' comments.
zeahmed Jul 13, 2018
d72a4ac
Addressed reviewers' comments.
zeahmed Jul 13, 2018
3dd9847
Updated OptionalColumnTransform comments.
zeahmed Jul 13, 2018
730fe93
Updated comments.
zeahmed Jul 13, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions src/Microsoft.ML.Transforms/GroupTransform.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -88,6 +88,18 @@ public sealed class Arguments : TransformInputBase

private readonly GroupSchema _schema;

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="groupKey">Columns to group by</param>
/// <param name="columns">Columns to group together</param>
public GroupTransform(IHostEnvironment env, IDataView input, string groupKey, params string[] columns)
: this(env, new Arguments() { GroupKey = new[] { groupKey }, Column = columns }, input)
{
}

public GroupTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, input)
{
Expand Down
35 changes: 31 additions & 4 deletions src/Microsoft.ML.Transforms/HashJoinTransform.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,14 @@ public sealed class HashJoinTransform : OneToOneTransformBase
public const int NumBitsMin = 1;
public const int NumBitsLim = 32;

private static class Defaults
{
public const bool Join = true;
public const int HashBits = NumBitsLim - 1;
public const uint Seed = 314489979;
public const bool Ordered = true;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)",
Expand All@@ -45,17 +53,17 @@ public sealed class Arguments : TransformInputBase
public Column[] Column;

[Argument(ArgumentType.AtMostOnce, HelpText = "Whether the values need to be combined for a single hash")]
public bool Join = true;
public bool Join = Defaults.Join;

[Argument(ArgumentType.AtMostOnce, HelpText = "Number of bits to hash into. Must be between 1 and 31, inclusive.",
ShortName = "bits", SortOrder = 2)]
public int HashBits = NumBitsLim - 1;
public int HashBits = Defaults.HashBits;

[Argument(ArgumentType.AtMostOnce, HelpText = "Hashing seed")]
public uint Seed = 314489979;
public uint Seed = Defaults.Seed;

[Argument(ArgumentType.AtMostOnce, HelpText = "Whether the position of each term should be included in the hash", ShortName = "ord")]
public bool Ordered = true;
public bool Ordered = Defaults.Ordered;
}

public sealed class Column : OneToOneColumn
Expand DownExpand Up@@ -166,6 +174,25 @@ private static VersionInfo GetVersionInfo()

private readonly ColumnInfoEx[] _exes;

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
/// <param name="join">Whether the values need to be combined for a single hash.</param>
/// <param name="hashBits">Number of bits to hash into. Must be between 1 and 31, inclusive.</param>
public HashJoinTransform(IHostEnvironment env,
IDataView input,
string name,
string source = null,
bool join = Defaults.Join,
int hashBits = Defaults.HashBits)
: this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, Join = join, HashBits = hashBits }, input)
{
}

public HashJoinTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, Contracts.CheckRef(args, nameof(args)).Column, input, TestColumnType)
{
Expand Down
12 changes: 12 additions & 0 deletions src/Microsoft.ML.Transforms/KeyToBinaryVectorTransform.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,18 @@ private static VersionInfo GetVersionInfo()

private readonly VectorType[] _types;

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
public KeyToBinaryVectorTransform(IHostEnvironment env, IDataView input, string name, string source = null)
: this(env, new Arguments() { Column = new[] { new KeyToVectorTransform.Column() { Source = source ?? name, Name = name } } }, input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand Down
19 changes: 19 additions & 0 deletions src/Microsoft.ML.Transforms/LoadTransform.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -39,6 +39,25 @@ public class Arguments

internal const string Summary = "Loads specified transforms from the model file and applies them to current data.";

/// <summary>
/// A helper method to create <see cref="LoadTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="modelFile">Model file to load the transforms from.</param>
/// <param name="tag">The tags (comma-separated) to be loaded (or omitted, if complement is true).</param>
/// <param name="complement">Whether to load all transforms except those marked by tags.</param>
public static IDataTransform Create(IHostEnvironment env, IDataView input, string modelFile, string[] tag, bool complement = false)
{
var args = new Arguments()
{
ModelFile = modelFile,
Tag = tag,
Complement = complement
};
return Create(env, args, input);
}

public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
Expand Down
39 changes: 36 additions & 3 deletions src/Microsoft.ML.Transforms/MutualInformationFeatureSelection.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -33,6 +33,13 @@ public static class MutualInformationFeatureSelectionTransform
public const string UserName = "Mutual Information Feature Selection Transform";
public const string ShortName = "MIFeatureSelection";

private static class Defaults
{
public const string LabelColumn = DefaultColumnNames.Label;
public const int SlotsInOutput = 1000;
public const int NumBins = 256;
}

public sealed class Arguments : TransformInputBase
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Columns to use for feature selection", ShortName = "col",
Expand All@@ -41,19 +48,45 @@ public sealed class Arguments : TransformInputBase

[Argument(ArgumentType.LastOccurenceWins, HelpText = "Column to use for labels", ShortName = "lab",
SortOrder = 4, Purpose = SpecialPurpose.ColumnName)]
public string LabelColumn = DefaultColumnNames.Label;
public string LabelColumn = Defaults.LabelColumn;

[Argument(ArgumentType.AtMostOnce, HelpText = "The maximum number of slots to preserve in output", ShortName = "topk,numSlotsToKeep",
SortOrder = 1)]
public int SlotsInOutput = 1000;
public int SlotsInOutput = Defaults.SlotsInOutput;

[Argument(ArgumentType.AtMostOnce, HelpText = "Max number of bins for R4/R8 columns, power of 2 recommended",
ShortName = "bins")]
public int NumBins = 256;
public int NumBins = Defaults.NumBins;
}

internal static string RegistrationName = "MutualInformationFeatureSelectionTransform";

/// <summary>
/// A helper method to create <see cref="IDataTransform"/> for selecting the top k slots ordered by their mutual information.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="labelColumn">Column to use for labels.</param>
/// <param name="slotsInOutput">The maximum number of slots to preserve in output.</param>
/// <param name="numBins">Max number of bins for R4/R8 columns, power of 2 recommended.</param>
/// <param name="columns">Columns to use for feature selection.</param>
public static IDataTransform Create(IHostEnvironment env,
IDataView input,
string labelColumn = Defaults.LabelColumn,
int slotsInOutput = Defaults.SlotsInOutput,
int numBins = Defaults.NumBins,
params string[] columns)
{
var args = new Arguments()
{
Column = columns,
LabelColumn = labelColumn,
SlotsInOutput = slotsInOutput,
NumBins = numBins
};
return Create(env, args, input);
}

/// <summary>
/// Create method corresponding to SignatureDataTransform.
/// </summary>
Expand Down
12 changes: 12 additions & 0 deletions src/Microsoft.ML.Transforms/NADropTransform.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -69,6 +69,18 @@ private static VersionInfo GetVersionInfo()
// The isNA delegates, parallel to Infos.
private readonly Delegate[] _isNAs;

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
public NADropTransform(IHostEnvironment env, IDataView input, string name, string source = null)
: this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } } }, input)
{
}

public NADropTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(Contracts.CheckRef(env, nameof(env)), RegistrationName, env.CheckRef(args, nameof(args)).Column, input, TestType)
{
Expand Down
36 changes: 36 additions & 0 deletions src/Microsoft.ML.Transforms/NAHandleTransform.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -36,10 +36,25 @@ public static class NAHandleTransform
{
public enum ReplacementKind
{
/// <summary>
/// Replace with the default value of the column based on it's type. For example, 'zero' for numeric and 'empty' for string/text columns.
/// </summary>
[EnumValueDisplay("Zero/empty")]
DefaultValue,

/// <summary>
/// Replace with the mean value of the column. Supports only numeric/time span/ DateTime columns.
/// </summary>
Mean,

/// <summary>
/// Replace with the minimum value of the column. Supports only numeric/time span/ DateTime columns.
/// </summary>
Minimum,

/// <summary>
/// Replace with the maximum value of the column. Supports only numeric/time span/ DateTime columns.
/// </summary>
Maximum,

[HideEnumValue]
Expand DownExpand Up@@ -105,6 +120,27 @@ public bool TryUnparse(StringBuilder sb)
internal const string FriendlyName = "NA Handle Transform";
internal const string ShortName = "NAHandle";

/// <summary>
/// A helper method to create <see cref="NAHandleTransform"/> for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
/// <param name="replaceWith">The replacement method to utilize.</param>
public static IDataTransform Create(IHostEnvironment env, IDataView input, string name, string source = null, ReplacementKind replaceWith = ReplacementKind.DefaultValue)
{
var args = new Arguments()
{
Column = new[]
{
new Column() { Source = source ?? name, Name = name }
},
ReplaceWith = replaceWith
};
return Create(env, args, input);
}

public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
Expand Down
12 changes: 12 additions & 0 deletions src/Microsoft.ML.Transforms/NAIndicatorTransform.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -85,6 +85,18 @@ private static string TestType(ColumnType type)
// The output column types, parallel to Infos.
private readonly ColumnType[] _types;

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
public NAIndicatorTransform(IHostEnvironment env, IDataView input, string name, string source = null)
: this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } } }, input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand Down
13 changes: 13 additions & 0 deletions src/Microsoft.ML.Transforms/NAReplaceTransform.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -186,6 +186,19 @@ private static string TestType<T>(ColumnType type)

public override bool CanSaveOnnx => true;

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
/// <param name="replacementKind">The replacement method to utilize.</param>
public NAReplaceTransform(IHostEnvironment env, IDataView input, string name, string source = null, ReplacementKind replacementKind = ReplacementKind.DefaultValue)
: this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, ReplacementKind = replacementKind }, input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand Down
20 changes: 19 additions & 1 deletion src/Microsoft.ML.Transforms/OptionalColumnTransform.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,7 +26,14 @@

namespace Microsoft.ML.Runtime.DataPipe
{
public class OptionalColumnTransform : RowToRowMapperTransformBase
/// <summary>
/// This transform is used to mark some of the columns (e.g. Label) optional during training so that the columns are not required during scoring.
/// When applied to new data, if any of the optional columns is not present a dummy columns is created having the same properties (e.g. 'name', 'type' etc.) as used during training.
/// The columns are filled with default values. The value is
/// - scalar for scalar column
/// - totally sparse vector for vector column.
/// </summary>
public sealed class OptionalColumnTransform : RowToRowMapperTransformBase
{
public sealed class Arguments : TransformInputBase
{
Expand DownExpand Up@@ -232,6 +239,17 @@ private static VersionInfo GetVersionInfo()

private const string RegistrationName = "OptionalColumn";

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="columns">Columns to transform.</param>
public OptionalColumnTransform(IHostEnvironment env, IDataView input, params string[] columns)
: this(env, new Arguments() { Column = columns }, input)
{
}

/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
Expand Down
27 changes: 25 additions & 2 deletions src/Microsoft.ML.Transforms/RffTransform.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -27,20 +27,26 @@ namespace Microsoft.ML.Runtime.Data

public sealed class RffTransform : OneToOneTransformBase
{
private static class Defaults
{
public const int NewDim = 1000;
public const bool UseSin = false;
}

public sealed class Arguments
{
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)]
public Column[] Column;

[Argument(ArgumentType.AtMostOnce, HelpText = "The number of random Fourier features to create", ShortName = "dim")]
public int NewDim = 1000;
public int NewDim = Defaults.NewDim;

[Argument(ArgumentType.Multiple, HelpText = "which kernel to use?", ShortName = "kernel")]
public SubComponent<IFourierDistributionSampler, SignatureFourierDistributionSampler> MatrixGenerator =
new SubComponent<IFourierDistributionSampler, SignatureFourierDistributionSampler>(GaussianFourierSampler.LoadName);

[Argument(ArgumentType.AtMostOnce, HelpText = "create two features for every random Fourier frequency? (one for cos and one for sin)")]
public bool UseSin = false;
public bool UseSin = Defaults.UseSin;

[Argument(ArgumentType.LastOccurenceWins,
HelpText = "The seed of the random number generator for generating the new features (if unspecified, " +
Expand DownExpand Up@@ -232,6 +238,23 @@ private static string TestColumnType(ColumnType type)
return "Expected R4 or vector of R4 with known size";
}

/// <summary>
/// Convenience constructor for public facing API.
/// </summary>
/// <param name="env">Host Environment.</param>
/// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
/// <param name="newDim">The number of random Fourier features to create.</param>
/// <param name="name">Name of the output column.</param>
/// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
public RffTransform(IHostEnvironment env,
IDataView input,
int newDim,
string name,
string source = null)
: this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, NewDim = newDim }, input)
{
}

/// <summary>
/// Public constructor corresponding to <see cref="SignatureDataTransform"/>.
/// </summary>
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
[Part 3] Added convenience constructors for set of transforms.#520

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

zeahmed
merged 6 commits into

from

Jul 16, 2018
 Merged


[Part 3] Added convenience constructors for set of transforms.
#520
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
d9c4a29

[Part 3] Added convenience constructors for set of transforms.


zeahmed Jul 10, 2018
6cd77eb

Addressed reviewers' comments.


zeahmed Jul 12, 2018
75a8b6d

Addressed reviewers' comments.


zeahmed Jul 13, 2018
d72a4ac

Addressed reviewers' comments.


zeahmed Jul 13, 2018
3dd9847

Updated OptionalColumnTransform comments.


zeahmed Jul 13, 2018
730fe93

Updated comments.


zeahmed Jul 13, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/GroupTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,6 +88,18 @@ public sealed class Arguments : TransformInputBase

 private readonly GroupSchema _schema;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="groupKey">Columns to group by</param>
 /// <param name="columns">Columns to group together</param>
 public GroupTransform(IHostEnvironment env, IDataView input, string groupKey, params string[] columns)
 : this(env, new Arguments() { GroupKey = new[] { groupKey }, Column = columns }, input)
 {
 }

 public GroupTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(env, RegistrationName, input)
 {
Expand Down

35 changes: 31 additions & 4 deletions
src/Microsoft.ML.Transforms/HashJoinTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,14 @@ public sealed class HashJoinTransform : OneToOneTransformBase
 public const int NumBitsMin = 1;
 public const int NumBitsLim = 32;

 private static class Defaults
 {
 public const bool Join = true;
 public const int HashBits = NumBitsLim - 1;
 public const uint Seed = 314489979;
 public const bool Ordered = true;
 }

 public sealed class Arguments : TransformInputBase
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)",
Expand All@@ -45,17 +53,17 @@ public sealed class Arguments : TransformInputBase
 public Column[] Column;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Whether the values need to be combined for a single hash")]
 public bool Join = true;
 public bool Join = Defaults.Join;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Number of bits to hash into. Must be between 1 and 31, inclusive.",
 ShortName = "bits", SortOrder = 2)]
 public int HashBits = NumBitsLim - 1;
 public int HashBits = Defaults.HashBits;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Hashing seed")]
 public uint Seed = 314489979;
 public uint Seed = Defaults.Seed;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Whether the position of each term should be included in the hash", ShortName = "ord")]
 public bool Ordered = true;
 public bool Ordered = Defaults.Ordered;
 }

 public sealed class Column : OneToOneColumn
Expand DownExpand Up@@ -166,6 +174,25 @@ private static VersionInfo GetVersionInfo()

 private readonly ColumnInfoEx[] _exes;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="join">Whether the values need to be combined for a single hash.</param>
 /// <param name="hashBits">Number of bits to hash into. Must be between 1 and 31, inclusive.</param>
 public HashJoinTransform(IHostEnvironment env,
 IDataView input,
 string name,
 string source = null,
 bool join = Defaults.Join,
 int hashBits = Defaults.HashBits)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, Join = join, HashBits = hashBits }, input)
 {
 }

 public HashJoinTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(env, RegistrationName, Contracts.CheckRef(args, nameof(args)).Column, input, TestColumnType)
 {
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/KeyToBinaryVectorTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,18 @@ private static VersionInfo GetVersionInfo()

 private readonly VectorType[] _types;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public KeyToBinaryVectorTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new KeyToVectorTransform.Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

19 changes: 19 additions & 0 deletions
src/Microsoft.ML.Transforms/LoadTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -39,6 +39,25 @@ public class Arguments

 internal const string Summary = "Loads specified transforms from the model file and applies them to current data.";

 /// <summary>
 /// A helper method to create <see cref="LoadTransform"/> for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="modelFile">Model file to load the transforms from.</param>
 /// <param name="tag">The tags (comma-separated) to be loaded (or omitted, if complement is true).</param>
 /// <param name="complement">Whether to load all transforms except those marked by tags.</param>
 public static IDataTransform Create(IHostEnvironment env, IDataView input, string modelFile, string[] tag, bool complement = false)
 {
 var args = new Arguments()
 {
 ModelFile = modelFile,
 Tag = tag,
 Complement = complement
 };
 return Create(env, args, input);
 }

 public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
 {
 Contracts.CheckValue(env, nameof(env));
Expand Down

39 changes: 36 additions & 3 deletions
src/Microsoft.ML.Transforms/MutualInformationFeatureSelection.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,6 +33,13 @@ public static class MutualInformationFeatureSelectionTransform
 public const string UserName = "Mutual Information Feature Selection Transform";
 public const string ShortName = "MIFeatureSelection";

 private static class Defaults
 {
 public const string LabelColumn = DefaultColumnNames.Label;
 public const int SlotsInOutput = 1000;
 public const int NumBins = 256;
 }

 public sealed class Arguments : TransformInputBase
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Columns to use for feature selection", ShortName = "col",
Expand All@@ -41,19 +48,45 @@ public sealed class Arguments : TransformInputBase

 [Argument(ArgumentType.LastOccurenceWins, HelpText = "Column to use for labels", ShortName = "lab",
 SortOrder = 4, Purpose = SpecialPurpose.ColumnName)]
 public string LabelColumn = DefaultColumnNames.Label;
 public string LabelColumn = Defaults.LabelColumn;

 [Argument(ArgumentType.AtMostOnce, HelpText = "The maximum number of slots to preserve in output", ShortName = "topk,numSlotsToKeep",
 SortOrder = 1)]
 public int SlotsInOutput = 1000;
 public int SlotsInOutput = Defaults.SlotsInOutput;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Max number of bins for R4/R8 columns, power of 2 recommended",
 ShortName = "bins")]
 public int NumBins = 256;
 public int NumBins = Defaults.NumBins;
 }

 internal static string RegistrationName = "MutualInformationFeatureSelectionTransform";

 /// <summary>
 /// A helper method to create <see cref="IDataTransform"/> for selecting the top k slots ordered by their mutual information.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="labelColumn">Column to use for labels.</param>
 /// <param name="slotsInOutput">The maximum number of slots to preserve in output.</param>
 /// <param name="numBins">Max number of bins for R4/R8 columns, power of 2 recommended.</param>
 /// <param name="columns">Columns to use for feature selection.</param>
 public static IDataTransform Create(IHostEnvironment env,
 IDataView input,
 string labelColumn = Defaults.LabelColumn,
 int slotsInOutput = Defaults.SlotsInOutput,
 int numBins = Defaults.NumBins,
 params string[] columns)
 {
 var args = new Arguments()
 {
 Column = columns,
 LabelColumn = labelColumn,
 SlotsInOutput = slotsInOutput,
 NumBins = numBins
 };
 return Create(env, args, input);
 }

 /// <summary>
 /// Create method corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/NADropTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,6 +69,18 @@ private static VersionInfo GetVersionInfo()
 // The isNA delegates, parallel to Infos.
 private readonly Delegate[] _isNAs;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public NADropTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 public NADropTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(Contracts.CheckRef(env, nameof(env)), RegistrationName, env.CheckRef(args, nameof(args)).Column, input, TestType)
 {
Expand Down

36 changes: 36 additions & 0 deletions
src/Microsoft.ML.Transforms/NAHandleTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,10 +36,25 @@ public static class NAHandleTransform
 {
 public enum ReplacementKind
 {
 /// <summary>
 /// Replace with the default value of the column based on it's type. For example, 'zero' for numeric and 'empty' for string/text columns.
 /// </summary>
 [EnumValueDisplay("Zero/empty")]
 DefaultValue,

 /// <summary>
 /// Replace with the mean value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Mean,

 /// <summary>
 /// Replace with the minimum value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Minimum,

 /// <summary>
 /// Replace with the maximum value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Maximum,

 [HideEnumValue]
Expand DownExpand Up@@ -105,6 +120,27 @@ public bool TryUnparse(StringBuilder sb)
 internal const string FriendlyName = "NA Handle Transform";
 internal const string ShortName = "NAHandle";

 /// <summary>
 /// A helper method to create <see cref="NAHandleTransform"/> for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="replaceWith">The replacement method to utilize.</param>
 public static IDataTransform Create(IHostEnvironment env, IDataView input, string name, string source = null, ReplacementKind replaceWith = ReplacementKind.DefaultValue)
 {
 var args = new Arguments()
 {
 Column = new[] 
 {
 new Column() { Source = source ?? name, Name = name }
 },
 ReplaceWith = replaceWith
 };
 return Create(env, args, input);
 }

 public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
 {
 Contracts.CheckValue(env, nameof(env));
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/NAIndicatorTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -85,6 +85,18 @@ private static string TestType(ColumnType type)
 // The output column types, parallel to Infos.
 private readonly ColumnType[] _types;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public NAIndicatorTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

13 changes: 13 additions & 0 deletions
src/Microsoft.ML.Transforms/NAReplaceTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -186,6 +186,19 @@ private static string TestType<T>(ColumnType type)

 public override bool CanSaveOnnx => true;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="replacementKind">The replacement method to utilize.</param>
 public NAReplaceTransform(IHostEnvironment env, IDataView input, string name, string source = null, ReplacementKind replacementKind = ReplacementKind.DefaultValue)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, ReplacementKind = replacementKind }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

20 changes: 19 additions & 1 deletion
src/Microsoft.ML.Transforms/OptionalColumnTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,7 +26,14 @@

namespace Microsoft.ML.Runtime.DataPipe
{
 public class OptionalColumnTransform : RowToRowMapperTransformBase
 /// <summary>
 /// This transform is used to mark some of the columns (e.g. Label) optional during training so that the columns are not required during scoring.
 /// When applied to new data, if any of the optional columns is not present a dummy columns is created having the same properties (e.g. 'name', 'type' etc.) as used during training.
 /// The columns are filled with default values. The value is
 /// - scalar for scalar column
 /// - totally sparse vector for vector column. 
 /// </summary>
 public sealed class OptionalColumnTransform : RowToRowMapperTransformBase
 {
 public sealed class Arguments : TransformInputBase
 {
Expand DownExpand Up@@ -232,6 +239,17 @@ private static VersionInfo GetVersionInfo()

 private const string RegistrationName = "OptionalColumn";

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="columns">Columns to transform.</param>
 public OptionalColumnTransform(IHostEnvironment env, IDataView input, params string[] columns)
 : this(env, new Arguments() { Column = columns }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

27 changes: 25 additions & 2 deletions
src/Microsoft.ML.Transforms/RffTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,20 +27,26 @@ namespace Microsoft.ML.Runtime.Data

 public sealed class RffTransform : OneToOneTransformBase
 {
 private static class Defaults
 {
 public const int NewDim = 1000;
 public const bool UseSin = false;
 }

 public sealed class Arguments
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)]
 public Column[] Column;

 [Argument(ArgumentType.AtMostOnce, HelpText = "The number of random Fourier features to create", ShortName = "dim")]
 public int NewDim = 1000;
 public int NewDim = Defaults.NewDim;

 [Argument(ArgumentType.Multiple, HelpText = "which kernel to use?", ShortName = "kernel")]
 public SubComponent<IFourierDistributionSampler, SignatureFourierDistributionSampler> MatrixGenerator =
 new SubComponent<IFourierDistributionSampler, SignatureFourierDistributionSampler>(GaussianFourierSampler.LoadName);

 [Argument(ArgumentType.AtMostOnce, HelpText = "create two features for every random Fourier frequency? (one for cos and one for sin)")]
 public bool UseSin = false;
 public bool UseSin = Defaults.UseSin;

 [Argument(ArgumentType.LastOccurenceWins,
 HelpText = "The seed of the random number generator for generating the new features (if unspecified, " +
Expand DownExpand Up@@ -232,6 +238,23 @@ private static string TestColumnType(ColumnType type)
 return "Expected R4 or vector of R4 with known size";
 }

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="newDim">The number of random Fourier features to create.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public RffTransform(IHostEnvironment env,
 IDataView input,
 int newDim,
 string name,
 string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, NewDim = newDim }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to <see cref="SignatureDataTransform"/>.
 /// </summary>
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[Part 3] Added convenience constructors for set of transforms.#520

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

zeahmed
merged 6 commits into

from

Jul 16, 2018
 Merged


[Part 3] Added convenience constructors for set of transforms.
#520
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
d9c4a29

[Part 3] Added convenience constructors for set of transforms.


zeahmed Jul 10, 2018
6cd77eb

Addressed reviewers' comments.


zeahmed Jul 12, 2018
75a8b6d

Addressed reviewers' comments.


zeahmed Jul 13, 2018
d72a4ac

Addressed reviewers' comments.


zeahmed Jul 13, 2018
3dd9847

Updated OptionalColumnTransform comments.


zeahmed Jul 13, 2018
730fe93

Updated comments.


zeahmed Jul 13, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/GroupTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,6 +88,18 @@ public sealed class Arguments : TransformInputBase

 private readonly GroupSchema _schema;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="groupKey">Columns to group by</param>
 /// <param name="columns">Columns to group together</param>
 public GroupTransform(IHostEnvironment env, IDataView input, string groupKey, params string[] columns)
 : this(env, new Arguments() { GroupKey = new[] { groupKey }, Column = columns }, input)
 {
 }

 public GroupTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(env, RegistrationName, input)
 {
Expand Down

35 changes: 31 additions & 4 deletions
src/Microsoft.ML.Transforms/HashJoinTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,14 @@ public sealed class HashJoinTransform : OneToOneTransformBase
 public const int NumBitsMin = 1;
 public const int NumBitsLim = 32;

 private static class Defaults
 {
 public const bool Join = true;
 public const int HashBits = NumBitsLim - 1;
 public const uint Seed = 314489979;
 public const bool Ordered = true;
 }

 public sealed class Arguments : TransformInputBase
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)",
Expand All@@ -45,17 +53,17 @@ public sealed class Arguments : TransformInputBase
 public Column[] Column;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Whether the values need to be combined for a single hash")]
 public bool Join = true;
 public bool Join = Defaults.Join;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Number of bits to hash into. Must be between 1 and 31, inclusive.",
 ShortName = "bits", SortOrder = 2)]
 public int HashBits = NumBitsLim - 1;
 public int HashBits = Defaults.HashBits;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Hashing seed")]
 public uint Seed = 314489979;
 public uint Seed = Defaults.Seed;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Whether the position of each term should be included in the hash", ShortName = "ord")]
 public bool Ordered = true;
 public bool Ordered = Defaults.Ordered;
 }

 public sealed class Column : OneToOneColumn
Expand DownExpand Up@@ -166,6 +174,25 @@ private static VersionInfo GetVersionInfo()

 private readonly ColumnInfoEx[] _exes;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="join">Whether the values need to be combined for a single hash.</param>
 /// <param name="hashBits">Number of bits to hash into. Must be between 1 and 31, inclusive.</param>
 public HashJoinTransform(IHostEnvironment env,
 IDataView input,
 string name,
 string source = null,
 bool join = Defaults.Join,
 int hashBits = Defaults.HashBits)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, Join = join, HashBits = hashBits }, input)
 {
 }

 public HashJoinTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(env, RegistrationName, Contracts.CheckRef(args, nameof(args)).Column, input, TestColumnType)
 {
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/KeyToBinaryVectorTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,18 @@ private static VersionInfo GetVersionInfo()

 private readonly VectorType[] _types;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public KeyToBinaryVectorTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new KeyToVectorTransform.Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

19 changes: 19 additions & 0 deletions
src/Microsoft.ML.Transforms/LoadTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -39,6 +39,25 @@ public class Arguments

 internal const string Summary = "Loads specified transforms from the model file and applies them to current data.";

 /// <summary>
 /// A helper method to create <see cref="LoadTransform"/> for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="modelFile">Model file to load the transforms from.</param>
 /// <param name="tag">The tags (comma-separated) to be loaded (or omitted, if complement is true).</param>
 /// <param name="complement">Whether to load all transforms except those marked by tags.</param>
 public static IDataTransform Create(IHostEnvironment env, IDataView input, string modelFile, string[] tag, bool complement = false)
 {
 var args = new Arguments()
 {
 ModelFile = modelFile,
 Tag = tag,
 Complement = complement
 };
 return Create(env, args, input);
 }

 public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
 {
 Contracts.CheckValue(env, nameof(env));
Expand Down

39 changes: 36 additions & 3 deletions
src/Microsoft.ML.Transforms/MutualInformationFeatureSelection.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,6 +33,13 @@ public static class MutualInformationFeatureSelectionTransform
 public const string UserName = "Mutual Information Feature Selection Transform";
 public const string ShortName = "MIFeatureSelection";

 private static class Defaults
 {
 public const string LabelColumn = DefaultColumnNames.Label;
 public const int SlotsInOutput = 1000;
 public const int NumBins = 256;
 }

 public sealed class Arguments : TransformInputBase
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Columns to use for feature selection", ShortName = "col",
Expand All@@ -41,19 +48,45 @@ public sealed class Arguments : TransformInputBase

 [Argument(ArgumentType.LastOccurenceWins, HelpText = "Column to use for labels", ShortName = "lab",
 SortOrder = 4, Purpose = SpecialPurpose.ColumnName)]
 public string LabelColumn = DefaultColumnNames.Label;
 public string LabelColumn = Defaults.LabelColumn;

 [Argument(ArgumentType.AtMostOnce, HelpText = "The maximum number of slots to preserve in output", ShortName = "topk,numSlotsToKeep",
 SortOrder = 1)]
 public int SlotsInOutput = 1000;
 public int SlotsInOutput = Defaults.SlotsInOutput;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Max number of bins for R4/R8 columns, power of 2 recommended",
 ShortName = "bins")]
 public int NumBins = 256;
 public int NumBins = Defaults.NumBins;
 }

 internal static string RegistrationName = "MutualInformationFeatureSelectionTransform";

 /// <summary>
 /// A helper method to create <see cref="IDataTransform"/> for selecting the top k slots ordered by their mutual information.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="labelColumn">Column to use for labels.</param>
 /// <param name="slotsInOutput">The maximum number of slots to preserve in output.</param>
 /// <param name="numBins">Max number of bins for R4/R8 columns, power of 2 recommended.</param>
 /// <param name="columns">Columns to use for feature selection.</param>
 public static IDataTransform Create(IHostEnvironment env,
 IDataView input,
 string labelColumn = Defaults.LabelColumn,
 int slotsInOutput = Defaults.SlotsInOutput,
 int numBins = Defaults.NumBins,
 params string[] columns)
 {
 var args = new Arguments()
 {
 Column = columns,
 LabelColumn = labelColumn,
 SlotsInOutput = slotsInOutput,
 NumBins = numBins
 };
 return Create(env, args, input);
 }

 /// <summary>
 /// Create method corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/NADropTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,6 +69,18 @@ private static VersionInfo GetVersionInfo()
 // The isNA delegates, parallel to Infos.
 private readonly Delegate[] _isNAs;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public NADropTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 public NADropTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(Contracts.CheckRef(env, nameof(env)), RegistrationName, env.CheckRef(args, nameof(args)).Column, input, TestType)
 {
Expand Down

36 changes: 36 additions & 0 deletions
src/Microsoft.ML.Transforms/NAHandleTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,10 +36,25 @@ public static class NAHandleTransform
 {
 public enum ReplacementKind
 {
 /// <summary>
 /// Replace with the default value of the column based on it's type. For example, 'zero' for numeric and 'empty' for string/text columns.
 /// </summary>
 [EnumValueDisplay("Zero/empty")]
 DefaultValue,

 /// <summary>
 /// Replace with the mean value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Mean,

 /// <summary>
 /// Replace with the minimum value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Minimum,

 /// <summary>
 /// Replace with the maximum value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Maximum,

 [HideEnumValue]
Expand DownExpand Up@@ -105,6 +120,27 @@ public bool TryUnparse(StringBuilder sb)
 internal const string FriendlyName = "NA Handle Transform";
 internal const string ShortName = "NAHandle";

 /// <summary>
 /// A helper method to create <see cref="NAHandleTransform"/> for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="replaceWith">The replacement method to utilize.</param>
 public static IDataTransform Create(IHostEnvironment env, IDataView input, string name, string source = null, ReplacementKind replaceWith = ReplacementKind.DefaultValue)
 {
 var args = new Arguments()
 {
 Column = new[] 
 {
 new Column() { Source = source ?? name, Name = name }
 },
 ReplaceWith = replaceWith
 };
 return Create(env, args, input);
 }

 public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
 {
 Contracts.CheckValue(env, nameof(env));
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/NAIndicatorTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -85,6 +85,18 @@ private static string TestType(ColumnType type)
 // The output column types, parallel to Infos.
 private readonly ColumnType[] _types;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public NAIndicatorTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

13 changes: 13 additions & 0 deletions
src/Microsoft.ML.Transforms/NAReplaceTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -186,6 +186,19 @@ private static string TestType<T>(ColumnType type)

 public override bool CanSaveOnnx => true;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="replacementKind">The replacement method to utilize.</param>
 public NAReplaceTransform(IHostEnvironment env, IDataView input, string name, string source = null, ReplacementKind replacementKind = ReplacementKind.DefaultValue)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, ReplacementKind = replacementKind }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

20 changes: 19 additions & 1 deletion
src/Microsoft.ML.Transforms/OptionalColumnTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,7 +26,14 @@

namespace Microsoft.ML.Runtime.DataPipe
{
 public class OptionalColumnTransform : RowToRowMapperTransformBase
 /// <summary>
 /// This transform is used to mark some of the columns (e.g. Label) optional during training so that the columns are not required during scoring.
 /// When applied to new data, if any of the optional columns is not present a dummy columns is created having the same properties (e.g. 'name', 'type' etc.) as used during training.
 /// The columns are filled with default values. The value is
 /// - scalar for scalar column
 /// - totally sparse vector for vector column. 
 /// </summary>
 public sealed class OptionalColumnTransform : RowToRowMapperTransformBase
 {
 public sealed class Arguments : TransformInputBase
 {
Expand DownExpand Up@@ -232,6 +239,17 @@ private static VersionInfo GetVersionInfo()

 private const string RegistrationName = "OptionalColumn";

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="columns">Columns to transform.</param>
 public OptionalColumnTransform(IHostEnvironment env, IDataView input, params string[] columns)
 : this(env, new Arguments() { Column = columns }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

27 changes: 25 additions & 2 deletions
src/Microsoft.ML.Transforms/RffTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,20 +27,26 @@ namespace Microsoft.ML.Runtime.Data

 public sealed class RffTransform : OneToOneTransformBase
 {
 private static class Defaults
 {
 public const int NewDim = 1000;
 public const bool UseSin = false;
 }

 public sealed class Arguments
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)]
 public Column[] Column;

 [Argument(ArgumentType.AtMostOnce, HelpText = "The number of random Fourier features to create", ShortName = "dim")]
 public int NewDim = 1000;
 public int NewDim = Defaults.NewDim;

 [Argument(ArgumentType.Multiple, HelpText = "which kernel to use?", ShortName = "kernel")]
 public SubComponent<IFourierDistributionSampler, SignatureFourierDistributionSampler> MatrixGenerator =
 new SubComponent<IFourierDistributionSampler, SignatureFourierDistributionSampler>(GaussianFourierSampler.LoadName);

 [Argument(ArgumentType.AtMostOnce, HelpText = "create two features for every random Fourier frequency? (one for cos and one for sin)")]
 public bool UseSin = false;
 public bool UseSin = Defaults.UseSin;

 [Argument(ArgumentType.LastOccurenceWins,
 HelpText = "The seed of the random number generator for generating the new features (if unspecified, " +
Expand DownExpand Up@@ -232,6 +238,23 @@ private static string TestColumnType(ColumnType type)
 return "Expected R4 or vector of R4 with known size";
 }

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="newDim">The number of random Fourier features to create.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public RffTransform(IHostEnvironment env,
 IDataView input,
 int newDim,
 string name,
 string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, NewDim = newDim }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to <see cref="SignatureDataTransform"/>.
 /// </summary>
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[Part 3] Added convenience constructors for set of transforms.#520

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

zeahmed
merged 6 commits into

from

Jul 16, 2018
 Merged


[Part 3] Added convenience constructors for set of transforms.
#520
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
d9c4a29

[Part 3] Added convenience constructors for set of transforms.


zeahmed Jul 10, 2018
6cd77eb

Addressed reviewers' comments.


zeahmed Jul 12, 2018
75a8b6d

Addressed reviewers' comments.


zeahmed Jul 13, 2018
d72a4ac

Addressed reviewers' comments.


zeahmed Jul 13, 2018
3dd9847

Updated OptionalColumnTransform comments.


zeahmed Jul 13, 2018
730fe93

Updated comments.


zeahmed Jul 13, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/GroupTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,6 +88,18 @@ public sealed class Arguments : TransformInputBase

 private readonly GroupSchema _schema;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="groupKey">Columns to group by</param>
 /// <param name="columns">Columns to group together</param>
 public GroupTransform(IHostEnvironment env, IDataView input, string groupKey, params string[] columns)
 : this(env, new Arguments() { GroupKey = new[] { groupKey }, Column = columns }, input)
 {
 }

 public GroupTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(env, RegistrationName, input)
 {
Expand Down

35 changes: 31 additions & 4 deletions
src/Microsoft.ML.Transforms/HashJoinTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,14 @@ public sealed class HashJoinTransform : OneToOneTransformBase
 public const int NumBitsMin = 1;
 public const int NumBitsLim = 32;

 private static class Defaults
 {
 public const bool Join = true;
 public const int HashBits = NumBitsLim - 1;
 public const uint Seed = 314489979;
 public const bool Ordered = true;
 }

 public sealed class Arguments : TransformInputBase
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)",
Expand All@@ -45,17 +53,17 @@ public sealed class Arguments : TransformInputBase
 public Column[] Column;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Whether the values need to be combined for a single hash")]
 public bool Join = true;
 public bool Join = Defaults.Join;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Number of bits to hash into. Must be between 1 and 31, inclusive.",
 ShortName = "bits", SortOrder = 2)]
 public int HashBits = NumBitsLim - 1;
 public int HashBits = Defaults.HashBits;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Hashing seed")]
 public uint Seed = 314489979;
 public uint Seed = Defaults.Seed;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Whether the position of each term should be included in the hash", ShortName = "ord")]
 public bool Ordered = true;
 public bool Ordered = Defaults.Ordered;
 }

 public sealed class Column : OneToOneColumn
Expand DownExpand Up@@ -166,6 +174,25 @@ private static VersionInfo GetVersionInfo()

 private readonly ColumnInfoEx[] _exes;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="join">Whether the values need to be combined for a single hash.</param>
 /// <param name="hashBits">Number of bits to hash into. Must be between 1 and 31, inclusive.</param>
 public HashJoinTransform(IHostEnvironment env,
 IDataView input,
 string name,
 string source = null,
 bool join = Defaults.Join,
 int hashBits = Defaults.HashBits)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, Join = join, HashBits = hashBits }, input)
 {
 }

 public HashJoinTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(env, RegistrationName, Contracts.CheckRef(args, nameof(args)).Column, input, TestColumnType)
 {
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/KeyToBinaryVectorTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,18 @@ private static VersionInfo GetVersionInfo()

 private readonly VectorType[] _types;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public KeyToBinaryVectorTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new KeyToVectorTransform.Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

19 changes: 19 additions & 0 deletions
src/Microsoft.ML.Transforms/LoadTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -39,6 +39,25 @@ public class Arguments

 internal const string Summary = "Loads specified transforms from the model file and applies them to current data.";

 /// <summary>
 /// A helper method to create <see cref="LoadTransform"/> for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="modelFile">Model file to load the transforms from.</param>
 /// <param name="tag">The tags (comma-separated) to be loaded (or omitted, if complement is true).</param>
 /// <param name="complement">Whether to load all transforms except those marked by tags.</param>
 public static IDataTransform Create(IHostEnvironment env, IDataView input, string modelFile, string[] tag, bool complement = false)
 {
 var args = new Arguments()
 {
 ModelFile = modelFile,
 Tag = tag,
 Complement = complement
 };
 return Create(env, args, input);
 }

 public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
 {
 Contracts.CheckValue(env, nameof(env));
Expand Down

39 changes: 36 additions & 3 deletions
src/Microsoft.ML.Transforms/MutualInformationFeatureSelection.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,6 +33,13 @@ public static class MutualInformationFeatureSelectionTransform
 public const string UserName = "Mutual Information Feature Selection Transform";
 public const string ShortName = "MIFeatureSelection";

 private static class Defaults
 {
 public const string LabelColumn = DefaultColumnNames.Label;
 public const int SlotsInOutput = 1000;
 public const int NumBins = 256;
 }

 public sealed class Arguments : TransformInputBase
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Columns to use for feature selection", ShortName = "col",
Expand All@@ -41,19 +48,45 @@ public sealed class Arguments : TransformInputBase

 [Argument(ArgumentType.LastOccurenceWins, HelpText = "Column to use for labels", ShortName = "lab",
 SortOrder = 4, Purpose = SpecialPurpose.ColumnName)]
 public string LabelColumn = DefaultColumnNames.Label;
 public string LabelColumn = Defaults.LabelColumn;

 [Argument(ArgumentType.AtMostOnce, HelpText = "The maximum number of slots to preserve in output", ShortName = "topk,numSlotsToKeep",
 SortOrder = 1)]
 public int SlotsInOutput = 1000;
 public int SlotsInOutput = Defaults.SlotsInOutput;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Max number of bins for R4/R8 columns, power of 2 recommended",
 ShortName = "bins")]
 public int NumBins = 256;
 public int NumBins = Defaults.NumBins;
 }

 internal static string RegistrationName = "MutualInformationFeatureSelectionTransform";

 /// <summary>
 /// A helper method to create <see cref="IDataTransform"/> for selecting the top k slots ordered by their mutual information.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="labelColumn">Column to use for labels.</param>
 /// <param name="slotsInOutput">The maximum number of slots to preserve in output.</param>
 /// <param name="numBins">Max number of bins for R4/R8 columns, power of 2 recommended.</param>
 /// <param name="columns">Columns to use for feature selection.</param>
 public static IDataTransform Create(IHostEnvironment env,
 IDataView input,
 string labelColumn = Defaults.LabelColumn,
 int slotsInOutput = Defaults.SlotsInOutput,
 int numBins = Defaults.NumBins,
 params string[] columns)
 {
 var args = new Arguments()
 {
 Column = columns,
 LabelColumn = labelColumn,
 SlotsInOutput = slotsInOutput,
 NumBins = numBins
 };
 return Create(env, args, input);
 }

 /// <summary>
 /// Create method corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/NADropTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,6 +69,18 @@ private static VersionInfo GetVersionInfo()
 // The isNA delegates, parallel to Infos.
 private readonly Delegate[] _isNAs;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public NADropTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 public NADropTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(Contracts.CheckRef(env, nameof(env)), RegistrationName, env.CheckRef(args, nameof(args)).Column, input, TestType)
 {
Expand Down

36 changes: 36 additions & 0 deletions
src/Microsoft.ML.Transforms/NAHandleTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,10 +36,25 @@ public static class NAHandleTransform
 {
 public enum ReplacementKind
 {
 /// <summary>
 /// Replace with the default value of the column based on it's type. For example, 'zero' for numeric and 'empty' for string/text columns.
 /// </summary>
 [EnumValueDisplay("Zero/empty")]
 DefaultValue,

 /// <summary>
 /// Replace with the mean value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Mean,

 /// <summary>
 /// Replace with the minimum value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Minimum,

 /// <summary>
 /// Replace with the maximum value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Maximum,

 [HideEnumValue]
Expand DownExpand Up@@ -105,6 +120,27 @@ public bool TryUnparse(StringBuilder sb)
 internal const string FriendlyName = "NA Handle Transform";
 internal const string ShortName = "NAHandle";

 /// <summary>
 /// A helper method to create <see cref="NAHandleTransform"/> for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="replaceWith">The replacement method to utilize.</param>
 public static IDataTransform Create(IHostEnvironment env, IDataView input, string name, string source = null, ReplacementKind replaceWith = ReplacementKind.DefaultValue)
 {
 var args = new Arguments()
 {
 Column = new[] 
 {
 new Column() { Source = source ?? name, Name = name }
 },
 ReplaceWith = replaceWith
 };
 return Create(env, args, input);
 }

 public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
 {
 Contracts.CheckValue(env, nameof(env));
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/NAIndicatorTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -85,6 +85,18 @@ private static string TestType(ColumnType type)
 // The output column types, parallel to Infos.
 private readonly ColumnType[] _types;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public NAIndicatorTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

13 changes: 13 additions & 0 deletions
src/Microsoft.ML.Transforms/NAReplaceTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -186,6 +186,19 @@ private static string TestType<T>(ColumnType type)

 public override bool CanSaveOnnx => true;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="replacementKind">The replacement method to utilize.</param>
 public NAReplaceTransform(IHostEnvironment env, IDataView input, string name, string source = null, ReplacementKind replacementKind = ReplacementKind.DefaultValue)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, ReplacementKind = replacementKind }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

20 changes: 19 additions & 1 deletion
src/Microsoft.ML.Transforms/OptionalColumnTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,7 +26,14 @@

namespace Microsoft.ML.Runtime.DataPipe
{
 public class OptionalColumnTransform : RowToRowMapperTransformBase
 /// <summary>
 /// This transform is used to mark some of the columns (e.g. Label) optional during training so that the columns are not required during scoring.
 /// When applied to new data, if any of the optional columns is not present a dummy columns is created having the same properties (e.g. 'name', 'type' etc.) as used during training.
 /// The columns are filled with default values. The value is
 /// - scalar for scalar column
 /// - totally sparse vector for vector column. 
 /// </summary>
 public sealed class OptionalColumnTransform : RowToRowMapperTransformBase
 {
 public sealed class Arguments : TransformInputBase
 {
Expand DownExpand Up@@ -232,6 +239,17 @@ private static VersionInfo GetVersionInfo()

 private const string RegistrationName = "OptionalColumn";

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="columns">Columns to transform.</param>
 public OptionalColumnTransform(IHostEnvironment env, IDataView input, params string[] columns)
 : this(env, new Arguments() { Column = columns }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

27 changes: 25 additions & 2 deletions
src/Microsoft.ML.Transforms/RffTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,20 +27,26 @@ namespace Microsoft.ML.Runtime.Data

 public sealed class RffTransform : OneToOneTransformBase
 {
 private static class Defaults
 {
 public const int NewDim = 1000;
 public const bool UseSin = false;
 }

 public sealed class Arguments
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)]
 public Column[] Column;

 [Argument(ArgumentType.AtMostOnce, HelpText = "The number of random Fourier features to create", ShortName = "dim")]
 public int NewDim = 1000;
 public int NewDim = Defaults.NewDim;

 [Argument(ArgumentType.Multiple, HelpText = "which kernel to use?", ShortName = "kernel")]
 public SubComponent<IFourierDistributionSampler, SignatureFourierDistributionSampler> MatrixGenerator =
 new SubComponent<IFourierDistributionSampler, SignatureFourierDistributionSampler>(GaussianFourierSampler.LoadName);

 [Argument(ArgumentType.AtMostOnce, HelpText = "create two features for every random Fourier frequency? (one for cos and one for sin)")]
 public bool UseSin = false;
 public bool UseSin = Defaults.UseSin;

 [Argument(ArgumentType.LastOccurenceWins,
 HelpText = "The seed of the random number generator for generating the new features (if unspecified, " +
Expand DownExpand Up@@ -232,6 +238,23 @@ private static string TestColumnType(ColumnType type)
 return "Expected R4 or vector of R4 with known size";
 }

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="newDim">The number of random Fourier features to create.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public RffTransform(IHostEnvironment env,
 IDataView input,
 int newDim,
 string name,
 string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, NewDim = newDim }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to <see cref="SignatureDataTransform"/>.
 /// </summary>
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
[Part 3] Added convenience constructors for set of transforms.#520

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

zeahmed
merged 6 commits into

from

Jul 16, 2018
 Merged


[Part 3] Added convenience constructors for set of transforms.
#520
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
d9c4a29

[Part 3] Added convenience constructors for set of transforms.


zeahmed Jul 10, 2018
6cd77eb

Addressed reviewers' comments.


zeahmed Jul 12, 2018
75a8b6d

Addressed reviewers' comments.


zeahmed Jul 13, 2018
d72a4ac

Addressed reviewers' comments.


zeahmed Jul 13, 2018
3dd9847

Updated OptionalColumnTransform comments.


zeahmed Jul 13, 2018
730fe93

Updated comments.


zeahmed Jul 13, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/GroupTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,6 +88,18 @@ public sealed class Arguments : TransformInputBase

 private readonly GroupSchema _schema;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="groupKey">Columns to group by</param>
 /// <param name="columns">Columns to group together</param>
 public GroupTransform(IHostEnvironment env, IDataView input, string groupKey, params string[] columns)
 : this(env, new Arguments() { GroupKey = new[] { groupKey }, Column = columns }, input)
 {
 }

 public GroupTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(env, RegistrationName, input)
 {
Expand Down

35 changes: 31 additions & 4 deletions
src/Microsoft.ML.Transforms/HashJoinTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,14 @@ public sealed class HashJoinTransform : OneToOneTransformBase
 public const int NumBitsMin = 1;
 public const int NumBitsLim = 32;

 private static class Defaults
 {
 public const bool Join = true;
 public const int HashBits = NumBitsLim - 1;
 public const uint Seed = 314489979;
 public const bool Ordered = true;
 }

 public sealed class Arguments : TransformInputBase
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)",
Expand All@@ -45,17 +53,17 @@ public sealed class Arguments : TransformInputBase
 public Column[] Column;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Whether the values need to be combined for a single hash")]
 public bool Join = true;
 public bool Join = Defaults.Join;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Number of bits to hash into. Must be between 1 and 31, inclusive.",
 ShortName = "bits", SortOrder = 2)]
 public int HashBits = NumBitsLim - 1;
 public int HashBits = Defaults.HashBits;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Hashing seed")]
 public uint Seed = 314489979;
 public uint Seed = Defaults.Seed;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Whether the position of each term should be included in the hash", ShortName = "ord")]
 public bool Ordered = true;
 public bool Ordered = Defaults.Ordered;
 }

 public sealed class Column : OneToOneColumn
Expand DownExpand Up@@ -166,6 +174,25 @@ private static VersionInfo GetVersionInfo()

 private readonly ColumnInfoEx[] _exes;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="join">Whether the values need to be combined for a single hash.</param>
 /// <param name="hashBits">Number of bits to hash into. Must be between 1 and 31, inclusive.</param>
 public HashJoinTransform(IHostEnvironment env,
 IDataView input,
 string name,
 string source = null,
 bool join = Defaults.Join,
 int hashBits = Defaults.HashBits)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, Join = join, HashBits = hashBits }, input)
 {
 }

 public HashJoinTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(env, RegistrationName, Contracts.CheckRef(args, nameof(args)).Column, input, TestColumnType)
 {
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/KeyToBinaryVectorTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,18 @@ private static VersionInfo GetVersionInfo()

 private readonly VectorType[] _types;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public KeyToBinaryVectorTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new KeyToVectorTransform.Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

19 changes: 19 additions & 0 deletions
src/Microsoft.ML.Transforms/LoadTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -39,6 +39,25 @@ public class Arguments

 internal const string Summary = "Loads specified transforms from the model file and applies them to current data.";

 /// <summary>
 /// A helper method to create <see cref="LoadTransform"/> for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="modelFile">Model file to load the transforms from.</param>
 /// <param name="tag">The tags (comma-separated) to be loaded (or omitted, if complement is true).</param>
 /// <param name="complement">Whether to load all transforms except those marked by tags.</param>
 public static IDataTransform Create(IHostEnvironment env, IDataView input, string modelFile, string[] tag, bool complement = false)
 {
 var args = new Arguments()
 {
 ModelFile = modelFile,
 Tag = tag,
 Complement = complement
 };
 return Create(env, args, input);
 }

 public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
 {
 Contracts.CheckValue(env, nameof(env));
Expand Down

39 changes: 36 additions & 3 deletions
src/Microsoft.ML.Transforms/MutualInformationFeatureSelection.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,6 +33,13 @@ public static class MutualInformationFeatureSelectionTransform
 public const string UserName = "Mutual Information Feature Selection Transform";
 public const string ShortName = "MIFeatureSelection";

 private static class Defaults
 {
 public const string LabelColumn = DefaultColumnNames.Label;
 public const int SlotsInOutput = 1000;
 public const int NumBins = 256;
 }

 public sealed class Arguments : TransformInputBase
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Columns to use for feature selection", ShortName = "col",
Expand All@@ -41,19 +48,45 @@ public sealed class Arguments : TransformInputBase

 [Argument(ArgumentType.LastOccurenceWins, HelpText = "Column to use for labels", ShortName = "lab",
 SortOrder = 4, Purpose = SpecialPurpose.ColumnName)]
 public string LabelColumn = DefaultColumnNames.Label;
 public string LabelColumn = Defaults.LabelColumn;

 [Argument(ArgumentType.AtMostOnce, HelpText = "The maximum number of slots to preserve in output", ShortName = "topk,numSlotsToKeep",
 SortOrder = 1)]
 public int SlotsInOutput = 1000;
 public int SlotsInOutput = Defaults.SlotsInOutput;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Max number of bins for R4/R8 columns, power of 2 recommended",
 ShortName = "bins")]
 public int NumBins = 256;
 public int NumBins = Defaults.NumBins;
 }

 internal static string RegistrationName = "MutualInformationFeatureSelectionTransform";

 /// <summary>
 /// A helper method to create <see cref="IDataTransform"/> for selecting the top k slots ordered by their mutual information.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="labelColumn">Column to use for labels.</param>
 /// <param name="slotsInOutput">The maximum number of slots to preserve in output.</param>
 /// <param name="numBins">Max number of bins for R4/R8 columns, power of 2 recommended.</param>
 /// <param name="columns">Columns to use for feature selection.</param>
 public static IDataTransform Create(IHostEnvironment env,
 IDataView input,
 string labelColumn = Defaults.LabelColumn,
 int slotsInOutput = Defaults.SlotsInOutput,
 int numBins = Defaults.NumBins,
 params string[] columns)
 {
 var args = new Arguments()
 {
 Column = columns,
 LabelColumn = labelColumn,
 SlotsInOutput = slotsInOutput,
 NumBins = numBins
 };
 return Create(env, args, input);
 }

 /// <summary>
 /// Create method corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/NADropTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,6 +69,18 @@ private static VersionInfo GetVersionInfo()
 // The isNA delegates, parallel to Infos.
 private readonly Delegate[] _isNAs;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public NADropTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 public NADropTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(Contracts.CheckRef(env, nameof(env)), RegistrationName, env.CheckRef(args, nameof(args)).Column, input, TestType)
 {
Expand Down

36 changes: 36 additions & 0 deletions
src/Microsoft.ML.Transforms/NAHandleTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,10 +36,25 @@ public static class NAHandleTransform
 {
 public enum ReplacementKind
 {
 /// <summary>
 /// Replace with the default value of the column based on it's type. For example, 'zero' for numeric and 'empty' for string/text columns.
 /// </summary>
 [EnumValueDisplay("Zero/empty")]
 DefaultValue,

 /// <summary>
 /// Replace with the mean value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Mean,

 /// <summary>
 /// Replace with the minimum value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Minimum,

 /// <summary>
 /// Replace with the maximum value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Maximum,

 [HideEnumValue]
Expand DownExpand Up@@ -105,6 +120,27 @@ public bool TryUnparse(StringBuilder sb)
 internal const string FriendlyName = "NA Handle Transform";
 internal const string ShortName = "NAHandle";

 /// <summary>
 /// A helper method to create <see cref="NAHandleTransform"/> for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="replaceWith">The replacement method to utilize.</param>
 public static IDataTransform Create(IHostEnvironment env, IDataView input, string name, string source = null, ReplacementKind replaceWith = ReplacementKind.DefaultValue)
 {
 var args = new Arguments()
 {
 Column = new[] 
 {
 new Column() { Source = source ?? name, Name = name }
 },
 ReplaceWith = replaceWith
 };
 return Create(env, args, input);
 }

 public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
 {
 Contracts.CheckValue(env, nameof(env));
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/NAIndicatorTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -85,6 +85,18 @@ private static string TestType(ColumnType type)
 // The output column types, parallel to Infos.
 private readonly ColumnType[] _types;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public NAIndicatorTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

13 changes: 13 additions & 0 deletions
src/Microsoft.ML.Transforms/NAReplaceTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -186,6 +186,19 @@ private static string TestType<T>(ColumnType type)

 public override bool CanSaveOnnx => true;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="replacementKind">The replacement method to utilize.</param>
 public NAReplaceTransform(IHostEnvironment env, IDataView input, string name, string source = null, ReplacementKind replacementKind = ReplacementKind.DefaultValue)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, ReplacementKind = replacementKind }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

20 changes: 19 additions & 1 deletion
src/Microsoft.ML.Transforms/OptionalColumnTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,7 +26,14 @@

namespace Microsoft.ML.Runtime.DataPipe
{
 public class OptionalColumnTransform : RowToRowMapperTransformBase
 /// <summary>
 /// This transform is used to mark some of the columns (e.g. Label) optional during training so that the columns are not required during scoring.
 /// When applied to new data, if any of the optional columns is not present a dummy columns is created having the same properties (e.g. 'name', 'type' etc.) as used during training.
 /// The columns are filled with default values. The value is
 /// - scalar for scalar column
 /// - totally sparse vector for vector column. 
 /// </summary>
 public sealed class OptionalColumnTransform : RowToRowMapperTransformBase
 {
 public sealed class Arguments : TransformInputBase
 {
Expand DownExpand Up@@ -232,6 +239,17 @@ private static VersionInfo GetVersionInfo()

 private const string RegistrationName = "OptionalColumn";

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="columns">Columns to transform.</param>
 public OptionalColumnTransform(IHostEnvironment env, IDataView input, params string[] columns)
 : this(env, new Arguments() { Column = columns }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

27 changes: 25 additions & 2 deletions
src/Microsoft.ML.Transforms/RffTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,20 +27,26 @@ namespace Microsoft.ML.Runtime.Data

 public sealed class RffTransform : OneToOneTransformBase
 {
 private static class Defaults
 {
 public const int NewDim = 1000;
 public const bool UseSin = false;
 }

 public sealed class Arguments
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)]
 public Column[] Column;

 [Argument(ArgumentType.AtMostOnce, HelpText = "The number of random Fourier features to create", ShortName = "dim")]
 public int NewDim = 1000;
 public int NewDim = Defaults.NewDim;

 [Argument(ArgumentType.Multiple, HelpText = "which kernel to use?", ShortName = "kernel")]
 public SubComponent<IFourierDistributionSampler, SignatureFourierDistributionSampler> MatrixGenerator =
 new SubComponent<IFourierDistributionSampler, SignatureFourierDistributionSampler>(GaussianFourierSampler.LoadName);

 [Argument(ArgumentType.AtMostOnce, HelpText = "create two features for every random Fourier frequency? (one for cos and one for sin)")]
 public bool UseSin = false;
 public bool UseSin = Defaults.UseSin;

 [Argument(ArgumentType.LastOccurenceWins,
 HelpText = "The seed of the random number generator for generating the new features (if unspecified, " +
Expand DownExpand Up@@ -232,6 +238,23 @@ private static string TestColumnType(ColumnType type)
 return "Expected R4 or vector of R4 with known size";
 }

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="newDim">The number of random Fourier features to create.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public RffTransform(IHostEnvironment env,
 IDataView input,
 int newDim,
 string name,
 string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, NewDim = newDim }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to <see cref="SignatureDataTransform"/>.
 /// </summary>
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[Part 3] Added convenience constructors for set of transforms.#520

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

zeahmed
merged 6 commits into

from

Jul 16, 2018
 Merged


[Part 3] Added convenience constructors for set of transforms.
#520
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
d9c4a29

[Part 3] Added convenience constructors for set of transforms.


zeahmed Jul 10, 2018
6cd77eb

Addressed reviewers' comments.


zeahmed Jul 12, 2018
75a8b6d

Addressed reviewers' comments.


zeahmed Jul 13, 2018
d72a4ac

Addressed reviewers' comments.


zeahmed Jul 13, 2018
3dd9847

Updated OptionalColumnTransform comments.


zeahmed Jul 13, 2018
730fe93

Updated comments.


zeahmed Jul 13, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/GroupTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,6 +88,18 @@ public sealed class Arguments : TransformInputBase

 private readonly GroupSchema _schema;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="groupKey">Columns to group by</param>
 /// <param name="columns">Columns to group together</param>
 public GroupTransform(IHostEnvironment env, IDataView input, string groupKey, params string[] columns)
 : this(env, new Arguments() { GroupKey = new[] { groupKey }, Column = columns }, input)
 {
 }

 public GroupTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(env, RegistrationName, input)
 {
Expand Down

35 changes: 31 additions & 4 deletions
src/Microsoft.ML.Transforms/HashJoinTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,14 @@ public sealed class HashJoinTransform : OneToOneTransformBase
 public const int NumBitsMin = 1;
 public const int NumBitsLim = 32;

 private static class Defaults
 {
 public const bool Join = true;
 public const int HashBits = NumBitsLim - 1;
 public const uint Seed = 314489979;
 public const bool Ordered = true;
 }

 public sealed class Arguments : TransformInputBase
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)",
Expand All@@ -45,17 +53,17 @@ public sealed class Arguments : TransformInputBase
 public Column[] Column;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Whether the values need to be combined for a single hash")]
 public bool Join = true;
 public bool Join = Defaults.Join;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Number of bits to hash into. Must be between 1 and 31, inclusive.",
 ShortName = "bits", SortOrder = 2)]
 public int HashBits = NumBitsLim - 1;
 public int HashBits = Defaults.HashBits;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Hashing seed")]
 public uint Seed = 314489979;
 public uint Seed = Defaults.Seed;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Whether the position of each term should be included in the hash", ShortName = "ord")]
 public bool Ordered = true;
 public bool Ordered = Defaults.Ordered;
 }

 public sealed class Column : OneToOneColumn
Expand DownExpand Up@@ -166,6 +174,25 @@ private static VersionInfo GetVersionInfo()

 private readonly ColumnInfoEx[] _exes;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="join">Whether the values need to be combined for a single hash.</param>
 /// <param name="hashBits">Number of bits to hash into. Must be between 1 and 31, inclusive.</param>
 public HashJoinTransform(IHostEnvironment env,
 IDataView input,
 string name,
 string source = null,
 bool join = Defaults.Join,
 int hashBits = Defaults.HashBits)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, Join = join, HashBits = hashBits }, input)
 {
 }

 public HashJoinTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(env, RegistrationName, Contracts.CheckRef(args, nameof(args)).Column, input, TestColumnType)
 {
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/KeyToBinaryVectorTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,18 @@ private static VersionInfo GetVersionInfo()

 private readonly VectorType[] _types;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public KeyToBinaryVectorTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new KeyToVectorTransform.Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

19 changes: 19 additions & 0 deletions
src/Microsoft.ML.Transforms/LoadTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -39,6 +39,25 @@ public class Arguments

 internal const string Summary = "Loads specified transforms from the model file and applies them to current data.";

 /// <summary>
 /// A helper method to create <see cref="LoadTransform"/> for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="modelFile">Model file to load the transforms from.</param>
 /// <param name="tag">The tags (comma-separated) to be loaded (or omitted, if complement is true).</param>
 /// <param name="complement">Whether to load all transforms except those marked by tags.</param>
 public static IDataTransform Create(IHostEnvironment env, IDataView input, string modelFile, string[] tag, bool complement = false)
 {
 var args = new Arguments()
 {
 ModelFile = modelFile,
 Tag = tag,
 Complement = complement
 };
 return Create(env, args, input);
 }

 public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
 {
 Contracts.CheckValue(env, nameof(env));
Expand Down

39 changes: 36 additions & 3 deletions
src/Microsoft.ML.Transforms/MutualInformationFeatureSelection.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,6 +33,13 @@ public static class MutualInformationFeatureSelectionTransform
 public const string UserName = "Mutual Information Feature Selection Transform";
 public const string ShortName = "MIFeatureSelection";

 private static class Defaults
 {
 public const string LabelColumn = DefaultColumnNames.Label;
 public const int SlotsInOutput = 1000;
 public const int NumBins = 256;
 }

 public sealed class Arguments : TransformInputBase
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Columns to use for feature selection", ShortName = "col",
Expand All@@ -41,19 +48,45 @@ public sealed class Arguments : TransformInputBase

 [Argument(ArgumentType.LastOccurenceWins, HelpText = "Column to use for labels", ShortName = "lab",
 SortOrder = 4, Purpose = SpecialPurpose.ColumnName)]
 public string LabelColumn = DefaultColumnNames.Label;
 public string LabelColumn = Defaults.LabelColumn;

 [Argument(ArgumentType.AtMostOnce, HelpText = "The maximum number of slots to preserve in output", ShortName = "topk,numSlotsToKeep",
 SortOrder = 1)]
 public int SlotsInOutput = 1000;
 public int SlotsInOutput = Defaults.SlotsInOutput;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Max number of bins for R4/R8 columns, power of 2 recommended",
 ShortName = "bins")]
 public int NumBins = 256;
 public int NumBins = Defaults.NumBins;
 }

 internal static string RegistrationName = "MutualInformationFeatureSelectionTransform";

 /// <summary>
 /// A helper method to create <see cref="IDataTransform"/> for selecting the top k slots ordered by their mutual information.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="labelColumn">Column to use for labels.</param>
 /// <param name="slotsInOutput">The maximum number of slots to preserve in output.</param>
 /// <param name="numBins">Max number of bins for R4/R8 columns, power of 2 recommended.</param>
 /// <param name="columns">Columns to use for feature selection.</param>
 public static IDataTransform Create(IHostEnvironment env,
 IDataView input,
 string labelColumn = Defaults.LabelColumn,
 int slotsInOutput = Defaults.SlotsInOutput,
 int numBins = Defaults.NumBins,
 params string[] columns)
 {
 var args = new Arguments()
 {
 Column = columns,
 LabelColumn = labelColumn,
 SlotsInOutput = slotsInOutput,
 NumBins = numBins
 };
 return Create(env, args, input);
 }

 /// <summary>
 /// Create method corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/NADropTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,6 +69,18 @@ private static VersionInfo GetVersionInfo()
 // The isNA delegates, parallel to Infos.
 private readonly Delegate[] _isNAs;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public NADropTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 public NADropTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(Contracts.CheckRef(env, nameof(env)), RegistrationName, env.CheckRef(args, nameof(args)).Column, input, TestType)
 {
Expand Down

36 changes: 36 additions & 0 deletions
src/Microsoft.ML.Transforms/NAHandleTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,10 +36,25 @@ public static class NAHandleTransform
 {
 public enum ReplacementKind
 {
 /// <summary>
 /// Replace with the default value of the column based on it's type. For example, 'zero' for numeric and 'empty' for string/text columns.
 /// </summary>
 [EnumValueDisplay("Zero/empty")]
 DefaultValue,

 /// <summary>
 /// Replace with the mean value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Mean,

 /// <summary>
 /// Replace with the minimum value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Minimum,

 /// <summary>
 /// Replace with the maximum value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Maximum,

 [HideEnumValue]
Expand DownExpand Up@@ -105,6 +120,27 @@ public bool TryUnparse(StringBuilder sb)
 internal const string FriendlyName = "NA Handle Transform";
 internal const string ShortName = "NAHandle";

 /// <summary>
 /// A helper method to create <see cref="NAHandleTransform"/> for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="replaceWith">The replacement method to utilize.</param>
 public static IDataTransform Create(IHostEnvironment env, IDataView input, string name, string source = null, ReplacementKind replaceWith = ReplacementKind.DefaultValue)
 {
 var args = new Arguments()
 {
 Column = new[] 
 {
 new Column() { Source = source ?? name, Name = name }
 },
 ReplaceWith = replaceWith
 };
 return Create(env, args, input);
 }

 public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
 {
 Contracts.CheckValue(env, nameof(env));
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/NAIndicatorTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -85,6 +85,18 @@ private static string TestType(ColumnType type)
 // The output column types, parallel to Infos.
 private readonly ColumnType[] _types;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public NAIndicatorTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

13 changes: 13 additions & 0 deletions
src/Microsoft.ML.Transforms/NAReplaceTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -186,6 +186,19 @@ private static string TestType<T>(ColumnType type)

 public override bool CanSaveOnnx => true;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="replacementKind">The replacement method to utilize.</param>
 public NAReplaceTransform(IHostEnvironment env, IDataView input, string name, string source = null, ReplacementKind replacementKind = ReplacementKind.DefaultValue)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, ReplacementKind = replacementKind }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

20 changes: 19 additions & 1 deletion
src/Microsoft.ML.Transforms/OptionalColumnTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,7 +26,14 @@

namespace Microsoft.ML.Runtime.DataPipe
{
 public class OptionalColumnTransform : RowToRowMapperTransformBase
 /// <summary>
 /// This transform is used to mark some of the columns (e.g. Label) optional during training so that the columns are not required during scoring.
 /// When applied to new data, if any of the optional columns is not present a dummy columns is created having the same properties (e.g. 'name', 'type' etc.) as used during training.
 /// The columns are filled with default values. The value is
 /// - scalar for scalar column
 /// - totally sparse vector for vector column. 
 /// </summary>
 public sealed class OptionalColumnTransform : RowToRowMapperTransformBase
 {
 public sealed class Arguments : TransformInputBase
 {
Expand DownExpand Up@@ -232,6 +239,17 @@ private static VersionInfo GetVersionInfo()

 private const string RegistrationName = "OptionalColumn";

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="columns">Columns to transform.</param>
 public OptionalColumnTransform(IHostEnvironment env, IDataView input, params string[] columns)
 : this(env, new Arguments() { Column = columns }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

27 changes: 25 additions & 2 deletions
src/Microsoft.ML.Transforms/RffTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,20 +27,26 @@ namespace Microsoft.ML.Runtime.Data

 public sealed class RffTransform : OneToOneTransformBase
 {
 private static class Defaults
 {
 public const int NewDim = 1000;
 public const bool UseSin = false;
 }

 public sealed class Arguments
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)]
 public Column[] Column;

 [Argument(ArgumentType.AtMostOnce, HelpText = "The number of random Fourier features to create", ShortName = "dim")]
 public int NewDim = 1000;
 public int NewDim = Defaults.NewDim;

 [Argument(ArgumentType.Multiple, HelpText = "which kernel to use?", ShortName = "kernel")]
 public SubComponent<IFourierDistributionSampler, SignatureFourierDistributionSampler> MatrixGenerator =
 new SubComponent<IFourierDistributionSampler, SignatureFourierDistributionSampler>(GaussianFourierSampler.LoadName);

 [Argument(ArgumentType.AtMostOnce, HelpText = "create two features for every random Fourier frequency? (one for cos and one for sin)")]
 public bool UseSin = false;
 public bool UseSin = Defaults.UseSin;

 [Argument(ArgumentType.LastOccurenceWins,
 HelpText = "The seed of the random number generator for generating the new features (if unspecified, " +
Expand DownExpand Up@@ -232,6 +238,23 @@ private static string TestColumnType(ColumnType type)
 return "Expected R4 or vector of R4 with known size";
 }

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="newDim">The number of random Fourier features to create.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public RffTransform(IHostEnvironment env,
 IDataView input,
 int newDim,
 string name,
 string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, NewDim = newDim }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to <see cref="SignatureDataTransform"/>.
 /// </summary>
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[Part 3] Added convenience constructors for set of transforms.#520

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

zeahmed
merged 6 commits into

from

Jul 16, 2018
 Merged


[Part 3] Added convenience constructors for set of transforms.
#520
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
d9c4a29

[Part 3] Added convenience constructors for set of transforms.


zeahmed Jul 10, 2018
6cd77eb

Addressed reviewers' comments.


zeahmed Jul 12, 2018
75a8b6d

Addressed reviewers' comments.


zeahmed Jul 13, 2018
d72a4ac

Addressed reviewers' comments.


zeahmed Jul 13, 2018
3dd9847

Updated OptionalColumnTransform comments.


zeahmed Jul 13, 2018
730fe93

Updated comments.


zeahmed Jul 13, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/GroupTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,6 +88,18 @@ public sealed class Arguments : TransformInputBase

 private readonly GroupSchema _schema;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="groupKey">Columns to group by</param>
 /// <param name="columns">Columns to group together</param>
 public GroupTransform(IHostEnvironment env, IDataView input, string groupKey, params string[] columns)
 : this(env, new Arguments() { GroupKey = new[] { groupKey }, Column = columns }, input)
 {
 }

 public GroupTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(env, RegistrationName, input)
 {
Expand Down

35 changes: 31 additions & 4 deletions
src/Microsoft.ML.Transforms/HashJoinTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,14 @@ public sealed class HashJoinTransform : OneToOneTransformBase
 public const int NumBitsMin = 1;
 public const int NumBitsLim = 32;

 private static class Defaults
 {
 public const bool Join = true;
 public const int HashBits = NumBitsLim - 1;
 public const uint Seed = 314489979;
 public const bool Ordered = true;
 }

 public sealed class Arguments : TransformInputBase
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)",
Expand All@@ -45,17 +53,17 @@ public sealed class Arguments : TransformInputBase
 public Column[] Column;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Whether the values need to be combined for a single hash")]
 public bool Join = true;
 public bool Join = Defaults.Join;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Number of bits to hash into. Must be between 1 and 31, inclusive.",
 ShortName = "bits", SortOrder = 2)]
 public int HashBits = NumBitsLim - 1;
 public int HashBits = Defaults.HashBits;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Hashing seed")]
 public uint Seed = 314489979;
 public uint Seed = Defaults.Seed;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Whether the position of each term should be included in the hash", ShortName = "ord")]
 public bool Ordered = true;
 public bool Ordered = Defaults.Ordered;
 }

 public sealed class Column : OneToOneColumn
Expand DownExpand Up@@ -166,6 +174,25 @@ private static VersionInfo GetVersionInfo()

 private readonly ColumnInfoEx[] _exes;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="join">Whether the values need to be combined for a single hash.</param>
 /// <param name="hashBits">Number of bits to hash into. Must be between 1 and 31, inclusive.</param>
 public HashJoinTransform(IHostEnvironment env,
 IDataView input,
 string name,
 string source = null,
 bool join = Defaults.Join,
 int hashBits = Defaults.HashBits)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, Join = join, HashBits = hashBits }, input)
 {
 }

 public HashJoinTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(env, RegistrationName, Contracts.CheckRef(args, nameof(args)).Column, input, TestColumnType)
 {
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/KeyToBinaryVectorTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,18 @@ private static VersionInfo GetVersionInfo()

 private readonly VectorType[] _types;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public KeyToBinaryVectorTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new KeyToVectorTransform.Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

19 changes: 19 additions & 0 deletions
src/Microsoft.ML.Transforms/LoadTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -39,6 +39,25 @@ public class Arguments

 internal const string Summary = "Loads specified transforms from the model file and applies them to current data.";

 /// <summary>
 /// A helper method to create <see cref="LoadTransform"/> for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="modelFile">Model file to load the transforms from.</param>
 /// <param name="tag">The tags (comma-separated) to be loaded (or omitted, if complement is true).</param>
 /// <param name="complement">Whether to load all transforms except those marked by tags.</param>
 public static IDataTransform Create(IHostEnvironment env, IDataView input, string modelFile, string[] tag, bool complement = false)
 {
 var args = new Arguments()
 {
 ModelFile = modelFile,
 Tag = tag,
 Complement = complement
 };
 return Create(env, args, input);
 }

 public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
 {
 Contracts.CheckValue(env, nameof(env));
Expand Down

39 changes: 36 additions & 3 deletions
src/Microsoft.ML.Transforms/MutualInformationFeatureSelection.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,6 +33,13 @@ public static class MutualInformationFeatureSelectionTransform
 public const string UserName = "Mutual Information Feature Selection Transform";
 public const string ShortName = "MIFeatureSelection";

 private static class Defaults
 {
 public const string LabelColumn = DefaultColumnNames.Label;
 public const int SlotsInOutput = 1000;
 public const int NumBins = 256;
 }

 public sealed class Arguments : TransformInputBase
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Columns to use for feature selection", ShortName = "col",
Expand All@@ -41,19 +48,45 @@ public sealed class Arguments : TransformInputBase

 [Argument(ArgumentType.LastOccurenceWins, HelpText = "Column to use for labels", ShortName = "lab",
 SortOrder = 4, Purpose = SpecialPurpose.ColumnName)]
 public string LabelColumn = DefaultColumnNames.Label;
 public string LabelColumn = Defaults.LabelColumn;

 [Argument(ArgumentType.AtMostOnce, HelpText = "The maximum number of slots to preserve in output", ShortName = "topk,numSlotsToKeep",
 SortOrder = 1)]
 public int SlotsInOutput = 1000;
 public int SlotsInOutput = Defaults.SlotsInOutput;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Max number of bins for R4/R8 columns, power of 2 recommended",
 ShortName = "bins")]
 public int NumBins = 256;
 public int NumBins = Defaults.NumBins;
 }

 internal static string RegistrationName = "MutualInformationFeatureSelectionTransform";

 /// <summary>
 /// A helper method to create <see cref="IDataTransform"/> for selecting the top k slots ordered by their mutual information.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="labelColumn">Column to use for labels.</param>
 /// <param name="slotsInOutput">The maximum number of slots to preserve in output.</param>
 /// <param name="numBins">Max number of bins for R4/R8 columns, power of 2 recommended.</param>
 /// <param name="columns">Columns to use for feature selection.</param>
 public static IDataTransform Create(IHostEnvironment env,
 IDataView input,
 string labelColumn = Defaults.LabelColumn,
 int slotsInOutput = Defaults.SlotsInOutput,
 int numBins = Defaults.NumBins,
 params string[] columns)
 {
 var args = new Arguments()
 {
 Column = columns,
 LabelColumn = labelColumn,
 SlotsInOutput = slotsInOutput,
 NumBins = numBins
 };
 return Create(env, args, input);
 }

 /// <summary>
 /// Create method corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/NADropTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,6 +69,18 @@ private static VersionInfo GetVersionInfo()
 // The isNA delegates, parallel to Infos.
 private readonly Delegate[] _isNAs;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public NADropTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 public NADropTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(Contracts.CheckRef(env, nameof(env)), RegistrationName, env.CheckRef(args, nameof(args)).Column, input, TestType)
 {
Expand Down

36 changes: 36 additions & 0 deletions
src/Microsoft.ML.Transforms/NAHandleTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,10 +36,25 @@ public static class NAHandleTransform
 {
 public enum ReplacementKind
 {
 /// <summary>
 /// Replace with the default value of the column based on it's type. For example, 'zero' for numeric and 'empty' for string/text columns.
 /// </summary>
 [EnumValueDisplay("Zero/empty")]
 DefaultValue,

 /// <summary>
 /// Replace with the mean value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Mean,

 /// <summary>
 /// Replace with the minimum value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Minimum,

 /// <summary>
 /// Replace with the maximum value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Maximum,

 [HideEnumValue]
Expand DownExpand Up@@ -105,6 +120,27 @@ public bool TryUnparse(StringBuilder sb)
 internal const string FriendlyName = "NA Handle Transform";
 internal const string ShortName = "NAHandle";

 /// <summary>
 /// A helper method to create <see cref="NAHandleTransform"/> for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="replaceWith">The replacement method to utilize.</param>
 public static IDataTransform Create(IHostEnvironment env, IDataView input, string name, string source = null, ReplacementKind replaceWith = ReplacementKind.DefaultValue)
 {
 var args = new Arguments()
 {
 Column = new[] 
 {
 new Column() { Source = source ?? name, Name = name }
 },
 ReplaceWith = replaceWith
 };
 return Create(env, args, input);
 }

 public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
 {
 Contracts.CheckValue(env, nameof(env));
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/NAIndicatorTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -85,6 +85,18 @@ private static string TestType(ColumnType type)
 // The output column types, parallel to Infos.
 private readonly ColumnType[] _types;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public NAIndicatorTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

13 changes: 13 additions & 0 deletions
src/Microsoft.ML.Transforms/NAReplaceTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -186,6 +186,19 @@ private static string TestType<T>(ColumnType type)

 public override bool CanSaveOnnx => true;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="replacementKind">The replacement method to utilize.</param>
 public NAReplaceTransform(IHostEnvironment env, IDataView input, string name, string source = null, ReplacementKind replacementKind = ReplacementKind.DefaultValue)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, ReplacementKind = replacementKind }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

20 changes: 19 additions & 1 deletion
src/Microsoft.ML.Transforms/OptionalColumnTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,7 +26,14 @@

namespace Microsoft.ML.Runtime.DataPipe
{
 public class OptionalColumnTransform : RowToRowMapperTransformBase
 /// <summary>
 /// This transform is used to mark some of the columns (e.g. Label) optional during training so that the columns are not required during scoring.
 /// When applied to new data, if any of the optional columns is not present a dummy columns is created having the same properties (e.g. 'name', 'type' etc.) as used during training.
 /// The columns are filled with default values. The value is
 /// - scalar for scalar column
 /// - totally sparse vector for vector column. 
 /// </summary>
 public sealed class OptionalColumnTransform : RowToRowMapperTransformBase
 {
 public sealed class Arguments : TransformInputBase
 {
Expand DownExpand Up@@ -232,6 +239,17 @@ private static VersionInfo GetVersionInfo()

 private const string RegistrationName = "OptionalColumn";

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="columns">Columns to transform.</param>
 public OptionalColumnTransform(IHostEnvironment env, IDataView input, params string[] columns)
 : this(env, new Arguments() { Column = columns }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

27 changes: 25 additions & 2 deletions
src/Microsoft.ML.Transforms/RffTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,20 +27,26 @@ namespace Microsoft.ML.Runtime.Data

 public sealed class RffTransform : OneToOneTransformBase
 {
 private static class Defaults
 {
 public const int NewDim = 1000;
 public const bool UseSin = false;
 }

 public sealed class Arguments
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)]
 public Column[] Column;

 [Argument(ArgumentType.AtMostOnce, HelpText = "The number of random Fourier features to create", ShortName = "dim")]
 public int NewDim = 1000;
 public int NewDim = Defaults.NewDim;

 [Argument(ArgumentType.Multiple, HelpText = "which kernel to use?", ShortName = "kernel")]
 public SubComponent<IFourierDistributionSampler, SignatureFourierDistributionSampler> MatrixGenerator =
 new SubComponent<IFourierDistributionSampler, SignatureFourierDistributionSampler>(GaussianFourierSampler.LoadName);

 [Argument(ArgumentType.AtMostOnce, HelpText = "create two features for every random Fourier frequency? (one for cos and one for sin)")]
 public bool UseSin = false;
 public bool UseSin = Defaults.UseSin;

 [Argument(ArgumentType.LastOccurenceWins,
 HelpText = "The seed of the random number generator for generating the new features (if unspecified, " +
Expand DownExpand Up@@ -232,6 +238,23 @@ private static string TestColumnType(ColumnType type)
 return "Expected R4 or vector of R4 with known size";
 }

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="newDim">The number of random Fourier features to create.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public RffTransform(IHostEnvironment env,
 IDataView input,
 int newDim,
 string name,
 string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, NewDim = newDim }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to <see cref="SignatureDataTransform"/>.
 /// </summary>
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
[Part 3] Added convenience constructors for set of transforms.#520

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

zeahmed
merged 6 commits into

from

Jul 16, 2018
 Merged


[Part 3] Added convenience constructors for set of transforms.
#520
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
d9c4a29

[Part 3] Added convenience constructors for set of transforms.


zeahmed Jul 10, 2018
6cd77eb

Addressed reviewers' comments.


zeahmed Jul 12, 2018
75a8b6d

Addressed reviewers' comments.


zeahmed Jul 13, 2018
d72a4ac

Addressed reviewers' comments.


zeahmed Jul 13, 2018
3dd9847

Updated OptionalColumnTransform comments.


zeahmed Jul 13, 2018
730fe93

Updated comments.


zeahmed Jul 13, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/GroupTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,6 +88,18 @@ public sealed class Arguments : TransformInputBase

 private readonly GroupSchema _schema;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="groupKey">Columns to group by</param>
 /// <param name="columns">Columns to group together</param>
 public GroupTransform(IHostEnvironment env, IDataView input, string groupKey, params string[] columns)
 : this(env, new Arguments() { GroupKey = new[] { groupKey }, Column = columns }, input)
 {
 }

 public GroupTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(env, RegistrationName, input)
 {
Expand Down

35 changes: 31 additions & 4 deletions
src/Microsoft.ML.Transforms/HashJoinTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,14 @@ public sealed class HashJoinTransform : OneToOneTransformBase
 public const int NumBitsMin = 1;
 public const int NumBitsLim = 32;

 private static class Defaults
 {
 public const bool Join = true;
 public const int HashBits = NumBitsLim - 1;
 public const uint Seed = 314489979;
 public const bool Ordered = true;
 }

 public sealed class Arguments : TransformInputBase
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)",
Expand All@@ -45,17 +53,17 @@ public sealed class Arguments : TransformInputBase
 public Column[] Column;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Whether the values need to be combined for a single hash")]
 public bool Join = true;
 public bool Join = Defaults.Join;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Number of bits to hash into. Must be between 1 and 31, inclusive.",
 ShortName = "bits", SortOrder = 2)]
 public int HashBits = NumBitsLim - 1;
 public int HashBits = Defaults.HashBits;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Hashing seed")]
 public uint Seed = 314489979;
 public uint Seed = Defaults.Seed;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Whether the position of each term should be included in the hash", ShortName = "ord")]
 public bool Ordered = true;
 public bool Ordered = Defaults.Ordered;
 }

 public sealed class Column : OneToOneColumn
Expand DownExpand Up@@ -166,6 +174,25 @@ private static VersionInfo GetVersionInfo()

 private readonly ColumnInfoEx[] _exes;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="join">Whether the values need to be combined for a single hash.</param>
 /// <param name="hashBits">Number of bits to hash into. Must be between 1 and 31, inclusive.</param>
 public HashJoinTransform(IHostEnvironment env,
 IDataView input,
 string name,
 string source = null,
 bool join = Defaults.Join,
 int hashBits = Defaults.HashBits)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, Join = join, HashBits = hashBits }, input)
 {
 }

 public HashJoinTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(env, RegistrationName, Contracts.CheckRef(args, nameof(args)).Column, input, TestColumnType)
 {
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/KeyToBinaryVectorTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,18 @@ private static VersionInfo GetVersionInfo()

 private readonly VectorType[] _types;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public KeyToBinaryVectorTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new KeyToVectorTransform.Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

19 changes: 19 additions & 0 deletions
src/Microsoft.ML.Transforms/LoadTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -39,6 +39,25 @@ public class Arguments

 internal const string Summary = "Loads specified transforms from the model file and applies them to current data.";

 /// <summary>
 /// A helper method to create <see cref="LoadTransform"/> for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="modelFile">Model file to load the transforms from.</param>
 /// <param name="tag">The tags (comma-separated) to be loaded (or omitted, if complement is true).</param>
 /// <param name="complement">Whether to load all transforms except those marked by tags.</param>
 public static IDataTransform Create(IHostEnvironment env, IDataView input, string modelFile, string[] tag, bool complement = false)
 {
 var args = new Arguments()
 {
 ModelFile = modelFile,
 Tag = tag,
 Complement = complement
 };
 return Create(env, args, input);
 }

 public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
 {
 Contracts.CheckValue(env, nameof(env));
Expand Down

39 changes: 36 additions & 3 deletions
src/Microsoft.ML.Transforms/MutualInformationFeatureSelection.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,6 +33,13 @@ public static class MutualInformationFeatureSelectionTransform
 public const string UserName = "Mutual Information Feature Selection Transform";
 public const string ShortName = "MIFeatureSelection";

 private static class Defaults
 {
 public const string LabelColumn = DefaultColumnNames.Label;
 public const int SlotsInOutput = 1000;
 public const int NumBins = 256;
 }

 public sealed class Arguments : TransformInputBase
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Columns to use for feature selection", ShortName = "col",
Expand All@@ -41,19 +48,45 @@ public sealed class Arguments : TransformInputBase

 [Argument(ArgumentType.LastOccurenceWins, HelpText = "Column to use for labels", ShortName = "lab",
 SortOrder = 4, Purpose = SpecialPurpose.ColumnName)]
 public string LabelColumn = DefaultColumnNames.Label;
 public string LabelColumn = Defaults.LabelColumn;

 [Argument(ArgumentType.AtMostOnce, HelpText = "The maximum number of slots to preserve in output", ShortName = "topk,numSlotsToKeep",
 SortOrder = 1)]
 public int SlotsInOutput = 1000;
 public int SlotsInOutput = Defaults.SlotsInOutput;

 [Argument(ArgumentType.AtMostOnce, HelpText = "Max number of bins for R4/R8 columns, power of 2 recommended",
 ShortName = "bins")]
 public int NumBins = 256;
 public int NumBins = Defaults.NumBins;
 }

 internal static string RegistrationName = "MutualInformationFeatureSelectionTransform";

 /// <summary>
 /// A helper method to create <see cref="IDataTransform"/> for selecting the top k slots ordered by their mutual information.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="labelColumn">Column to use for labels.</param>
 /// <param name="slotsInOutput">The maximum number of slots to preserve in output.</param>
 /// <param name="numBins">Max number of bins for R4/R8 columns, power of 2 recommended.</param>
 /// <param name="columns">Columns to use for feature selection.</param>
 public static IDataTransform Create(IHostEnvironment env,
 IDataView input,
 string labelColumn = Defaults.LabelColumn,
 int slotsInOutput = Defaults.SlotsInOutput,
 int numBins = Defaults.NumBins,
 params string[] columns)
 {
 var args = new Arguments()
 {
 Column = columns,
 LabelColumn = labelColumn,
 SlotsInOutput = slotsInOutput,
 NumBins = numBins
 };
 return Create(env, args, input);
 }

 /// <summary>
 /// Create method corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/NADropTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,6 +69,18 @@ private static VersionInfo GetVersionInfo()
 // The isNA delegates, parallel to Infos.
 private readonly Delegate[] _isNAs;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public NADropTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 public NADropTransform(IHostEnvironment env, Arguments args, IDataView input)
 : base(Contracts.CheckRef(env, nameof(env)), RegistrationName, env.CheckRef(args, nameof(args)).Column, input, TestType)
 {
Expand Down

36 changes: 36 additions & 0 deletions
src/Microsoft.ML.Transforms/NAHandleTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,10 +36,25 @@ public static class NAHandleTransform
 {
 public enum ReplacementKind
 {
 /// <summary>
 /// Replace with the default value of the column based on it's type. For example, 'zero' for numeric and 'empty' for string/text columns.
 /// </summary>
 [EnumValueDisplay("Zero/empty")]
 DefaultValue,

 /// <summary>
 /// Replace with the mean value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Mean,

 /// <summary>
 /// Replace with the minimum value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Minimum,

 /// <summary>
 /// Replace with the maximum value of the column. Supports only numeric/time span/ DateTime columns.
 /// </summary>
 Maximum,

 [HideEnumValue]
Expand DownExpand Up@@ -105,6 +120,27 @@ public bool TryUnparse(StringBuilder sb)
 internal const string FriendlyName = "NA Handle Transform";
 internal const string ShortName = "NAHandle";

 /// <summary>
 /// A helper method to create <see cref="NAHandleTransform"/> for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="replaceWith">The replacement method to utilize.</param>
 public static IDataTransform Create(IHostEnvironment env, IDataView input, string name, string source = null, ReplacementKind replaceWith = ReplacementKind.DefaultValue)
 {
 var args = new Arguments()
 {
 Column = new[] 
 {
 new Column() { Source = source ?? name, Name = name }
 },
 ReplaceWith = replaceWith
 };
 return Create(env, args, input);
 }

 public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
 {
 Contracts.CheckValue(env, nameof(env));
Expand Down

12 changes: 12 additions & 0 deletions
src/Microsoft.ML.Transforms/NAIndicatorTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -85,6 +85,18 @@ private static string TestType(ColumnType type)
 // The output column types, parallel to Infos.
 private readonly ColumnType[] _types;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public NAIndicatorTransform(IHostEnvironment env, IDataView input, string name, string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } } }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

13 changes: 13 additions & 0 deletions
src/Microsoft.ML.Transforms/NAReplaceTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -186,6 +186,19 @@ private static string TestType<T>(ColumnType type)

 public override bool CanSaveOnnx => true;

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="replacementKind">The replacement method to utilize.</param>
 public NAReplaceTransform(IHostEnvironment env, IDataView input, string name, string source = null, ReplacementKind replacementKind = ReplacementKind.DefaultValue)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, ReplacementKind = replacementKind }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

20 changes: 19 additions & 1 deletion
src/Microsoft.ML.Transforms/OptionalColumnTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,7 +26,14 @@

namespace Microsoft.ML.Runtime.DataPipe
{
 public class OptionalColumnTransform : RowToRowMapperTransformBase
 /// <summary>
 /// This transform is used to mark some of the columns (e.g. Label) optional during training so that the columns are not required during scoring.
 /// When applied to new data, if any of the optional columns is not present a dummy columns is created having the same properties (e.g. 'name', 'type' etc.) as used during training.
 /// The columns are filled with default values. The value is
 /// - scalar for scalar column
 /// - totally sparse vector for vector column. 
 /// </summary>
 public sealed class OptionalColumnTransform : RowToRowMapperTransformBase
 {
 public sealed class Arguments : TransformInputBase
 {
Expand DownExpand Up@@ -232,6 +239,17 @@ private static VersionInfo GetVersionInfo()

 private const string RegistrationName = "OptionalColumn";

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="columns">Columns to transform.</param>
 public OptionalColumnTransform(IHostEnvironment env, IDataView input, params string[] columns)
 : this(env, new Arguments() { Column = columns }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to SignatureDataTransform.
 /// </summary>
Expand Down

27 changes: 25 additions & 2 deletions
src/Microsoft.ML.Transforms/RffTransform.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,20 +27,26 @@ namespace Microsoft.ML.Runtime.Data

 public sealed class RffTransform : OneToOneTransformBase
 {
 private static class Defaults
 {
 public const int NewDim = 1000;
 public const bool UseSin = false;
 }

 public sealed class Arguments
 {
 [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)]
 public Column[] Column;

 [Argument(ArgumentType.AtMostOnce, HelpText = "The number of random Fourier features to create", ShortName = "dim")]
 public int NewDim = 1000;
 public int NewDim = Defaults.NewDim;

 [Argument(ArgumentType.Multiple, HelpText = "which kernel to use?", ShortName = "kernel")]
 public SubComponent<IFourierDistributionSampler, SignatureFourierDistributionSampler> MatrixGenerator =
 new SubComponent<IFourierDistributionSampler, SignatureFourierDistributionSampler>(GaussianFourierSampler.LoadName);

 [Argument(ArgumentType.AtMostOnce, HelpText = "create two features for every random Fourier frequency? (one for cos and one for sin)")]
 public bool UseSin = false;
 public bool UseSin = Defaults.UseSin;

 [Argument(ArgumentType.LastOccurenceWins,
 HelpText = "The seed of the random number generator for generating the new features (if unspecified, " +
Expand DownExpand Up@@ -232,6 +238,23 @@ private static string TestColumnType(ColumnType type)
 return "Expected R4 or vector of R4 with known size";
 }

 /// <summary>
 /// Convenience constructor for public facing API.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="newDim">The number of random Fourier features to create.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 public RffTransform(IHostEnvironment env,
 IDataView input,
 int newDim,
 string name,
 string source = null)
 : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } }, NewDim = newDim }, input)
 {
 }

 /// <summary>
 /// Public constructor corresponding to <see cref="SignatureDataTransform"/>.
 /// </summary>
Expand Down
Loading